From afec145edca8b8b75a7386ff4480049b120bde0e Mon Sep 17 00:00:00 2001 From: shrey Date: Fri, 30 Oct 2026 22:52:25 -0700 Subject: [PATCH 1/6] feat(passthrough): let a WebSocket relay bill a session from its client text (#49281) --- .../pass_through_endpoints.py | 17 ++++ .../pass_through_endpoints.py | 16 ++- .../test_pass_through_endpoints.py | 97 +++++++++++++++++++ 3 files changed, 129 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index cfac28db1a9..647e145aac1 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -148,6 +148,8 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, EndpointType, PassthroughStandardLoggingPayload, + WebSocketSessionMeter, + WebSocketSessionUsage, ) from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, CallTypes, Usage @@ -2595,6 +2597,13 @@ def _with_trace_context(headers: Mapping[str, str], parent_span: object) -> dict return inject_trace_context(headers, parent_span=parent_span) +def _log_session_usage(logging_obj: LiteLLMLoggingObj, usage: WebSocketSessionUsage) -> None: + logging_obj.model = usage.model # rebind-ok: the spend logger reads model and cost off the shared logging object + logging_obj.model_call_details["model"] = usage.model # rebind-ok: same shared logging object + logging_obj.model_call_details["custom_llm_provider"] = usage.custom_llm_provider # rebind-ok: same + logging_obj.model_call_details["response_cost"] = usage.response_cost # rebind-ok: same + + async def websocket_passthrough_request( websocket: WebSocket, target: str, @@ -2607,6 +2616,7 @@ async def websocket_passthrough_request( setup_model_rewriter: Callable[[str], str] | None = None, builtin_provider_route: bool = False, model: str | None = None, + meter: WebSocketSessionMeter | None = None, ): """ WebSocket passthrough request handler. @@ -2622,6 +2632,7 @@ async def websocket_passthrough_request( setup_model_rewriter: Optional rewrite of the setup frame's model before it reaches the upstream builtin_provider_route: Whether a built-in provider route serves the connection, so its URL names the provider model: The model the caller named before connecting, logged on a failure before the setup frame arrives + meter: Prices the session from the client text frames delivered upstream, for an upstream that reports no usage """ from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj @@ -2632,6 +2643,7 @@ async def websocket_passthrough_request( # Initialize tracking variables start_time: Final = datetime.now() websocket_messages: Final[list[dict[str, object]]] = [] + metered_total = 0 # rebind-ok: forward_client_to_upstream adds the measure of each client text frame it delivers litellm_call_id: Final = str(uuid.uuid4()) verbose_proxy_logger.info("WebSocket passthrough (%s): Starting WebSocket connection to %s", endpoint, target) @@ -2750,6 +2762,7 @@ async def websocket_passthrough_request( async def forward_client_to_upstream() -> None: """Forward messages from client to upstream WebSocket""" + nonlocal metered_total try: while True: message = await websocket.receive() @@ -2819,6 +2832,8 @@ async def websocket_passthrough_request( # Not a JSON message or doesn't contain setup data await upstream_ws.send(_rewrite_vertex_live_setup_model(text_data, setup_model_rewriter)) + if meter is not None: + metered_total += meter.measure(text_data) elif bytes_data is not None: await upstream_ws.send(bytes_data) except asyncio.CancelledError: @@ -2945,6 +2960,8 @@ async def websocket_passthrough_request( self.url = target_url mock_response: Final = MockWebSocketResponse(target) + if meter is not None: + _log_session_usage(logging_obj, meter.price(metered_total)) # Use the same success handler as HTTP passthrough endpoints GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( diff --git a/litellm/types/passthrough_endpoints/pass_through_endpoints.py b/litellm/types/passthrough_endpoints/pass_through_endpoints.py index 0d99cd54a6b..2f269dc87d9 100644 --- a/litellm/types/passthrough_endpoints/pass_through_endpoints.py +++ b/litellm/types/passthrough_endpoints/pass_through_endpoints.py @@ -1,5 +1,6 @@ +from dataclasses import dataclass from enum import Enum -from typing import Final +from typing import Final, Protocol from pydantic import Field from typing_extensions import TypedDict @@ -75,6 +76,19 @@ class PassthroughStandardLoggingPayload(TypedDict, total=False): """ +@dataclass(frozen=True, slots=True) +class WebSocketSessionUsage: + model: str + custom_llm_provider: str + response_cost: float | None + + +class WebSocketSessionMeter(Protocol): + def measure(self, client_text: str) -> int: ... + + def price(self, total: int) -> WebSocketSessionUsage: ... + + class PassThroughGuardrailSettings(LiteLLMPydanticObjectBase): """ Settings for a specific guardrail on a passthrough endpoint. diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index a9098ac3d59..b7e76ce0068 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -77,6 +77,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, EndpointType, PassthroughStandardLoggingPayload, + WebSocketSessionUsage, ) from tests._master_key import MASTER_KEY from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -7736,6 +7737,102 @@ async def test_websocket_passthrough_sends_deepgram_audio_bytes_and_control_text upstream_ws.close.assert_awaited_once() +@dataclass(frozen=True, slots=True) +class _LengthMeter: + rate: float + + def measure(self, client_text: str) -> int: + return len(client_text) + + def price(self, total: int) -> WebSocketSessionUsage: + return WebSocketSessionUsage(model="voice-1", custom_llm_provider="tts-provider", response_cost=total * self.rate) + + +@pytest.mark.asyncio +async def test_websocket_passthrough_logs_the_meters_price_for_the_client_text_it_delivered(): + """Text frames reach the upstream unchanged and are measured, binary frames are not, and the session is logged at + the model, provider and cost the meter prices for the total.""" + frames = ("hello", b"\x00\x01binary", "world!") + upstream_ws = RecordingUpstreamWebSocket() + websocket = _client_websocket( + AsyncMock( + side_effect=[ + *( + {"type": "websocket.receive", "bytes": frame} + if isinstance(frame, bytes) + else {"type": "websocket.receive", "text": frame} + for frame in frames + ), + {"type": "websocket.disconnect"}, + ] + ) + ) + + with ( + _patched_websocket_passthrough_environment(upstream_ws), + patch( # test-quality-ok: pass_through_endpoint_logging is a module global read inside websocket_passthrough_request; there is no injection seam + "litellm.proxy.pass_through_endpoints.pass_through_endpoints." + "pass_through_endpoint_logging.pass_through_async_success_handler", + new=AsyncMock(), + ) as success_handler, + ): + await websocket_passthrough_request( + websocket=websocket, + target="wss://upstream.example/v1/stream", + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth(), + forward_headers=False, + endpoint="/provider/v1/stream", + accept_websocket=False, + meter=_LengthMeter(rate=0.5), + ) + + assert [call.args[0] for call in upstream_ws.send.await_args_list] == list(frames) + logging_obj = success_handler.call_args.kwargs["logging_obj"] + assert logging_obj.model == "voice-1" + assert logging_obj.model_call_details["model"] == "voice-1" + assert logging_obj.model_call_details["custom_llm_provider"] == "tts-provider" + assert logging_obj.model_call_details["response_cost"] == (len("hello") + len("world!")) * 0.5 + + +@pytest.mark.asyncio +async def test_websocket_passthrough_meters_only_client_text_the_upstream_accepted(): + upstream_ws = RecordingUpstreamWebSocket() + upstream_ws.send = AsyncMock(side_effect=[None, ConnectionError("upstream went away")]) + websocket = _client_websocket( + AsyncMock( + side_effect=[ + {"type": "websocket.receive", "text": "hello"}, + {"type": "websocket.receive", "text": "never delivered"}, + {"type": "websocket.disconnect"}, + ] + ) + ) + + with ( + _patched_websocket_passthrough_environment(upstream_ws), + patch( # test-quality-ok: pass_through_endpoint_logging is a module global read inside websocket_passthrough_request; there is no injection seam + "litellm.proxy.pass_through_endpoints.pass_through_endpoints." + "pass_through_endpoint_logging.pass_through_async_success_handler", + new=AsyncMock(), + ) as success_handler, + ): + await websocket_passthrough_request( + websocket=websocket, + target="wss://upstream.example/v1/stream", + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth(), + forward_headers=False, + endpoint="/provider/v1/stream", + accept_websocket=False, + meter=_LengthMeter(rate=0.5), + ) + + logging_obj = success_handler.call_args.kwargs["logging_obj"] + assert [call.args[0] for call in upstream_ws.send.await_args_list] == ["hello", "never delivered"] + assert logging_obj.model_call_details["response_cost"] == len("hello") * 0.5 + + @pytest.mark.asyncio async def test_websocket_passthrough_vertex_live_setup_ack_names_the_model_but_is_not_billed_as_usage(): """Vertex Live keeps its special first frame: the setup acknowledgement is forwarded verbatim, read for the From d1d6d550631d0aaa3a2dc3eeaf48e7d3379cbcec Mon Sep 17 00:00:00 2001 From: mika-lt <349045450+mika-lt@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:11:34 -0700 Subject: [PATCH 2/6] fix(hosted_vllm): accept the vLLM production-stack router's file and batch objects (#49068) * fix(files): default a missing file object type to "file" so OpenAI-compatible servers that omit it stop 500ing * fix(batches): accept the vLLM production-stack router's batch object The router leaves out the batch object's "object" field and reports "pending" and "running" statuses, so batch create and retrieve through a hosted_vllm deployment failed validation with a 500. A missing or null object now reads as "batch", pending maps to validating and running to in_progress, and any other unknown value is still rejected * fix(batches): return OpenAI batch objects from batch list behind the vLLM router * fix(batches): type the batch list conversion without a TypeVar and refuse an async client on the sync path * fix(batches): keep batch list items as OpenAI's Batch so an odd item never fails a list * refactor(types): reuse the shared JSON value alias on the batch type * fix: return the provider page for an async client in sync batch list and pass through non-string list statuses * test: inject the SDK's own transport in the sync batch list test so it passes on OpenAI SDK 3.x --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/llms/openai/openai.py | 22 ++- litellm/types/llms/openai.py | 7 +- litellm/types/utils.py | 30 ++- tests/unit/batches/test_main.py | 186 +++++++++++++++++- tests/unit/files/test_main.py | 56 ++++++ tests/unit/llms/openai/test_openai.py | 41 ++++ .../unit/types/llms/test_types_llms_openai.py | 17 ++ 7 files changed, 349 insertions(+), 10 deletions(-) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index bdd9f285823..6e22ef9112e 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -16,6 +16,7 @@ from openai._base_client import make_request_options from openai._constants import RAW_RESPONSE_HEADER from openai._legacy_response import LegacyAPIResponse from openai._types import RequestOptions +from openai.pagination import SyncCursorPage from openai.types import CreateEmbeddingResponse from openai.types.beta.assistant_deleted import AssistantDeleted from openai.types.chat import ChatCompletion, ChatCompletionChunk @@ -40,6 +41,7 @@ from litellm.types.utils import ( LiteLLMBatch, ModelResponse, ModelResponseStream, + openai_batch_status, ) from litellm.utils import ( CustomStreamWrapper, @@ -2182,6 +2184,18 @@ class OpenAIFilesAPI(BaseLLM): return response +def _with_openai_batch_fields(batches: Sequence[Batch]) -> Sequence[Batch]: + return [ + batch.model_copy( + update={ + "object": batch.object or "batch", + "status": openai_batch_status(batch.status), + } + ) + for batch in batches + ] + + class OpenAIBatchesAPI(BaseLLM): """ OpenAI methods to support for batches @@ -2372,7 +2386,7 @@ class OpenAIBatchesAPI(BaseLLM): ): verbose_logger.debug("listing batches, after= %s, limit= %s", after, limit) response: Final = await openai_client.batches.list(after=after, limit=limit) - return response + return response.model_copy(update={"data": _with_openai_batch_fields(response.data)}) def list_batches( self, @@ -2384,7 +2398,7 @@ class OpenAIBatchesAPI(BaseLLM): organization: str | None, after: str | None = None, limit: int | None = None, - client: OpenAI | None = None, + client: OpenAI | AsyncOpenAI | None = None, litellm_params: Mapping[str, object] | None = None, ): openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client( @@ -2409,7 +2423,9 @@ class OpenAIBatchesAPI(BaseLLM): ) return self.alist_batches(openai_client=openai_client, after=after, limit=limit) response: Final = openai_client.batches.list(after=after, limit=limit) - return response + if not isinstance(response, SyncCursorPage): + return response + return response.model_copy(update={"data": _with_openai_batch_fields(response.data)}) class OpenAIAssistantsAPI(BaseLLM): diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index a54d66c4beb..ee86ec52327 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -402,7 +402,7 @@ class OpenAIFileObject(LiteLLMBaseModel): filename: str """The name of the file.""" - object: Literal["file"] + object: Literal["file"] = "file" """The object type, which is always `file`.""" purpose: OpenAIFilesPurpose @@ -440,6 +440,11 @@ class OpenAIFileObject(LiteLLMBaseModel): _hidden_params: dict = PrivateAttr(default={"response_cost": 0.0}) # no cost for writing a file + @field_validator("object", mode="before") + @classmethod + def _default_missing_object_type(cls, value: _JsonValue) -> _JsonValue: + return "file" if value is None else value + @property def hidden_params(self) -> dict[str, builtins.object]: # mutable-ok: API requires mutation return self._hidden_params diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 0c3440e88f7..2f6918d0744 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -106,6 +106,7 @@ from .llms.openai import ( OpenAIRealtimeStreamList, ResponsesAPIResponse, WebSearchOptions, + _JsonValue, ) from .rerank import RerankResponse as RerankResponse @@ -4963,29 +4964,50 @@ def _batch_usage_from_chat_usage(usage: Usage) -> BatchUsage: ) +OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS: Final = MappingProxyType( + {"pending": "validating", "running": "in_progress"} +) + + +def openai_batch_status(status: _JsonValue) -> _JsonValue: + return OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(status, status) if isinstance(status, str) else status + + class LiteLLMBatch(Batch): _hidden_params: dict = {} + object: Literal["batch"] = "batch" + + @field_validator("object", mode="before") + @classmethod + def _default_missing_object_type(cls, value: _JsonValue) -> _JsonValue: + return "batch" if value is None else value + + @field_validator("status", mode="before") + @classmethod + def _map_vllm_router_status(cls, value: _JsonValue) -> _JsonValue: + return openai_batch_status(value) + @property - def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation + def hidden_params(self) -> dict[str, _JsonValue]: # mutable-ok: API requires mutation return self._hidden_params @hidden_params.setter - def hidden_params(self, hidden_params: dict[str, object]) -> None: # mutable-ok: API requires mutation + def hidden_params(self, hidden_params: dict[str, _JsonValue]) -> None: # mutable-ok: API requires mutation self._hidden_params = hidden_params usage: Usage | None = None @field_validator("usage", mode="before") @classmethod - def _usage_from_batch_token_counts(cls, value: object) -> object: + def _usage_from_batch_token_counts(cls, value: _JsonValue) -> _JsonValue: counts: Final = _parse_batch_token_counts(value) return value if counts is None else _usage_with_batch_token_counts(Usage.model_validate(value), counts) @field_serializer("usage") def _serialize_usage_with_batch_fields( self, usage: Usage | None, info: FieldSerializationInfo - ) -> Mapping[str, object] | None: + ) -> Mapping[str, _JsonValue] | None: if not isinstance(usage, Usage): return usage chat_fields: Final = usage.model_dump( diff --git a/tests/unit/batches/test_main.py b/tests/unit/batches/test_main.py index fb97e0e4919..8b22395f462 100644 --- a/tests/unit/batches/test_main.py +++ b/tests/unit/batches/test_main.py @@ -37,13 +37,15 @@ import litellm.batches.main as bm import asyncio import datetime import json -from collections.abc import Mapping +from collections.abc import Awaitable, Callable, Mapping, Sequence from typing import Final import httpx import respx -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict from litellm.integrations.custom_logger import CustomLogger +from openai.types.batch import Batch +from litellm.types.utils import LiteLLMBatch # --------------------------------------------------------------------------- # @@ -1204,3 +1206,183 @@ async def test_delete_batch_output_file(respx_mock: respx.MockRouter, monkeypatc assert delete_route.call_count == 1 assert delete_response.id == "file-output123" assert delete_response.deleted is True + + +VLLM_ROUTER_API_BASE: Final = "http://vllm-router.test/v1" +VLLM_ROUTER_BATCH: Final = MappingProxyType( + { + "id": "batch_5f8dac", + "status": "pending", + "input_file_id": "file-b73f4c", + "created_at": 1793239207, + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + "output_file_id": None, + "total_requests": None, + "completed_requests": 0, + "failed_requests": 0, + "metadata": {}, + } +) + + +async def _create_on_vllm_router() -> LiteLLMBatch: + return await litellm.acreate_batch( + completion_window="24h", + endpoint="/v1/chat/completions", + input_file_id="file-b73f4c", + custom_llm_provider="hosted_vllm", + api_key="sk-test", + api_base=VLLM_ROUTER_API_BASE, + ) + + +async def _retrieve_from_vllm_router() -> LiteLLMBatch: + return await litellm.aretrieve_batch( + batch_id="batch_5f8dac", custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE + ) + + +VLLM_ROUTER_BATCH_CALLS: Final = ( + pytest.param("POST", "/batches", _create_on_vllm_router, id="acreate_batch"), + pytest.param("GET", "/batches/batch_5f8dac", _retrieve_from_vllm_router, id="aretrieve_batch"), +) + + +@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_BATCH_CALLS) +@pytest.mark.parametrize( + "router_status, openai_status", + ( + ("pending", "validating"), + ("running", "in_progress"), + ("completed", "completed"), + ("failed", "failed"), + ("cancelled", "cancelled"), + ), +) +async def test_hosted_vllm_router_batch_comes_back_as_an_openai_batch( + respx_mock: respx.MockRouter, + method: str, + path: str, + call_router: Callable[[], Awaitable[LiteLLMBatch]], + router_status: str, + openai_status: str, +): + respx_mock.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond( + 200, json={**VLLM_ROUTER_BATCH, "status": router_status} + ) + + batch: Final = await call_router() + + assert (batch.id, batch.object, batch.status) == ("batch_5f8dac", "batch", openai_status) + + +@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_BATCH_CALLS) +@pytest.mark.parametrize("field, value", (("status", "queued"), ("object", "file"))) +async def test_hosted_vllm_router_batch_with_an_unknown_value_is_rejected( + respx_mock: respx.MockRouter, + method: str, + path: str, + call_router: Callable[[], Awaitable[LiteLLMBatch]], + field: str, + value: str, +): + respx_mock.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond( + 200, json={**VLLM_ROUTER_BATCH, field: value} + ) + + with pytest.raises(ValidationError) as exc_info: + await call_router() + + assert [error["loc"] for error in exc_info.value.errors()] == [(field,)] + + +async def _list_on_vllm_router() -> Sequence[Batch]: + page: Final = await litellm.alist_batches( + custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE + ) + return page.data + + +async def _list_on_vllm_router_sync() -> Sequence[Batch]: + return litellm.list_batches(custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE).data + + +@pytest.mark.parametrize("list_router", (_list_on_vllm_router, _list_on_vllm_router_sync)) +async def test_hosted_vllm_router_batch_list_comes_back_as_openai_batches( + respx_mock: respx.MockRouter, list_router: Callable[[], Awaitable[Sequence[Batch]]] +): + respx_mock.get(f"{VLLM_ROUTER_API_BASE}/batches").respond( + 200, + json={ + "object": "list", + "data": [ + {**VLLM_ROUTER_BATCH, "id": "batch_5f8dac", "status": "pending"}, + {**VLLM_ROUTER_BATCH, "id": "batch_c74802", "status": "running"}, + ], + "first_id": "batch_5f8dac", + "last_id": "batch_c74802", + "has_more": False, + }, + ) + + batches: Final = await list_router() + + assert [(batch.id, batch.object, batch.status) for batch in batches] == [ + ("batch_5f8dac", "batch", "validating"), + ("batch_c74802", "batch", "in_progress"), + ] + + +async def test_hosted_vllm_router_batch_list_passes_through_a_status_that_is_not_a_string( + respx_mock: respx.MockRouter, +): + respx_mock.get(f"{VLLM_ROUTER_API_BASE}/batches").respond( + 200, + json={ + "object": "list", + "data": [{**VLLM_ROUTER_BATCH, "status": ["pending"]}], + "first_id": "batch_5f8dac", + "last_id": "batch_5f8dac", + "has_more": False, + }, + ) + + batches: Final = await _list_on_vllm_router() + + assert [(batch.id, batch.object, batch.status) for batch in batches] == [("batch_5f8dac", "batch", ["pending"])] + + +async def test_openai_batch_list_items_come_back_as_openai_sent_them(respx_mock: respx.MockRouter): + respx_mock.get("http://openai.test/v1/batches").respond( + 200, + json={ + "object": "list", + "data": [ + { + **_OPENAI_BATCH_JSON, + "status": "completed", + "usage": { + "input_tokens": 10, + "input_tokens_details": {"cached_tokens": 4}, + "output_tokens": 5, + "output_tokens_details": {"reasoning_tokens": 2}, + "total_tokens": 15, + }, + }, + {**_OPENAI_BATCH_JSON, "id": "batch_def456", "status": "queued"}, + ], + "first_id": "batch_abc123", + "last_id": "batch_def456", + "has_more": False, + }, + ) + + page: Final = await litellm.alist_batches( + custom_llm_provider="openai", api_key="sk-test", api_base="http://openai.test/v1" + ) + + first, second = page.data + assert first.usage is not None + assert (first.usage.input_tokens_details.cached_tokens, first.usage.output_tokens_details.reasoning_tokens) == (4, 2) + assert (second.id, second.object, second.status) == ("batch_def456", "batch", "queued") diff --git a/tests/unit/files/test_main.py b/tests/unit/files/test_main.py index e91951fb5ba..44aadc9014e 100644 --- a/tests/unit/files/test_main.py +++ b/tests/unit/files/test_main.py @@ -1,3 +1,4 @@ +from collections.abc import Awaitable, Callable from types import MappingProxyType from typing import Final from urllib.parse import parse_qs, urlparse @@ -124,6 +125,61 @@ async def test_afile_retrieve_rejects_a_provider_file_without_its_size(): assert [error["loc"] for error in exc_info.value.errors()] == [("bytes",)] +VLLM_ROUTER_API_BASE: Final = "http://vllm-router.test/v1" +VLLM_ROUTER_FILE: Final = MappingProxyType( + {"id": "file-b73f4c", "bytes": 156, "created_at": 1793238707, "filename": "batch_requests.jsonl", "purpose": "batch"} +) + + +async def _upload_to_vllm_router() -> OpenAIFileObject: + return await litellm.acreate_file( + file=("batch_requests.jsonl", b'{"custom_id": "request-1"}'), + purpose="batch", + custom_llm_provider="hosted_vllm", + api_key="sk-test", + api_base=VLLM_ROUTER_API_BASE, + ) + + +async def _retrieve_from_vllm_router() -> OpenAIFileObject: + return await litellm.afile_retrieve( + file_id="file-b73f4c", custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE + ) + + +VLLM_ROUTER_FILE_CALLS: Final = ( + pytest.param("POST", "/files", _upload_to_vllm_router, id="acreate_file"), + pytest.param("GET", "/files/file-b73f4c", _retrieve_from_vllm_router, id="afile_retrieve"), +) + + +@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_FILE_CALLS) +@respx.mock +async def test_hosted_vllm_file_without_an_object_type_comes_back_as_a_file( + method: str, path: str, call_router: Callable[[], Awaitable[OpenAIFileObject]] +): + respx.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond(200, json=dict(VLLM_ROUTER_FILE)) + + file_object: Final = await call_router() + + assert file_object.model_dump(exclude_none=True) == {**VLLM_ROUTER_FILE, "object": "file"} + + +@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_FILE_CALLS) +@respx.mock +async def test_hosted_vllm_file_with_a_different_object_type_is_rejected( + method: str, path: str, call_router: Callable[[], Awaitable[OpenAIFileObject]] +): + respx.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond( + 200, json={**VLLM_ROUTER_FILE, "object": "batch"} + ) + + with pytest.raises(ValidationError) as exc_info: + await call_router() + + assert [error["loc"] for error in exc_info.value.errors()] == [("object",)] + + _FILE_BODY: Final = b'{"prompt": "Hello", "completion": "Hi"}' _FINE_TUNE_FILE_JSON: Final = MappingProxyType( { diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index 3e1cbad9308..77a7c3b83fd 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -24,6 +24,7 @@ from litellm.llms.openai.openai import ( AsyncAssistantEventHandler, AsyncCursorPage, MessageData, + OpenAIBatchesAPI, OpenAIChatCompletion, OpenAIConfig, OpenAIMessage as Message, @@ -2320,3 +2321,43 @@ def test_openai_config_hoists_a_later_developer_message_only_for_a_model_off_ope model_info=None, ) assert [message["role"] for message in messages] == expected_roles + + +@pytest.mark.asyncio +async def test_sync_batch_list_given_an_async_client_returns_a_page_the_caller_can_await( + http_backend: ModuleType, +): + def respond(request: _Request) -> httpx.Response | SDKResponse: + assert str(request.url).startswith("http://vllm-router.test/v1/batches") + return http_backend.Response( + 200, + json={ + "object": "list", + "data": [ + { + "id": "batch_5f8dac", + "status": "pending", + "input_file_id": "file-b73f4c", + "created_at": 1793239207, + "endpoint": "/v1/chat/completions", + "completion_window": "24h", + } + ], + "first_id": "batch_5f8dac", + "last_id": "batch_5f8dac", + "has_more": False, + }, + ) + + async with http_backend.AsyncClient(transport=http_backend.MockTransport(respond)) as http_client: + page: Final = OpenAIBatchesAPI().list_batches( + _is_async=False, + api_key="sk-test", + api_base="http://vllm-router.test/v1", + timeout=5.0, + max_retries=0, + organization=None, + client=AsyncOpenAI(api_key="sk-test", base_url="http://vllm-router.test/v1", http_client=http_client), + ) + + assert [(batch.id, batch.status) for batch in (await page).data] == [("batch_5f8dac", "pending")] diff --git a/tests/unit/types/llms/test_types_llms_openai.py b/tests/unit/types/llms/test_types_llms_openai.py index 1b7a9693f7b..05801c9feec 100644 --- a/tests/unit/types/llms/test_types_llms_openai.py +++ b/tests/unit/types/llms/test_types_llms_openai.py @@ -607,6 +607,23 @@ def test_openai_file_object_accepts_pending_status(): assert file_obj.status == "pending" +@pytest.mark.parametrize("object_type", ({}, {"object": None}), ids=["missing", "null"]) +def test_openai_file_object_without_an_object_type_validates_as_a_file(object_type: dict[str, None]) -> None: + from litellm.types.llms.openai import OpenAIFileObject + + server_file: Final = { + "id": "file-b73f4c", + "bytes": 156, + "created_at": 1793238707, + "filename": "batch_requests.jsonl", + "purpose": "batch", + } + + file_obj: Final = OpenAIFileObject.model_validate({**server_file, **object_type}) + + assert file_obj.model_dump(exclude_none=True) == {**server_file, "object": "file"} + + class TestOpenAIFileObjectBatchGuardrailSerialization: """The proxy-only `litellm_batch_guardrail` key must reach the wire only when something set it.""" From a392a39627f51bf0369944556c22268ee8cffe83 Mon Sep 17 00:00:00 2001 From: simonp9 <349053457+simonp9@users.noreply.github.com> Date: Fri, 30 Oct 2026 23:14:51 -0700 Subject: [PATCH 3/6] fix(vertex_ai): bill Claude web search per request on deployments priced in model_info (#49160) * fix(vertex_ai): bill Claude web search per request on deployments priced in model_info A vertex_ai deployment priced in its model_info is registered under its hashed id, so the web search cost check on the model key never saw "claude" and billed Claude searches with the Gemini grounding calculator: one search per call, or Gemini's $0.035 default with no search price. The check now also reads the model the response names. * fix(vertex_ai): bill token-priced Claude deployments at the served model's search price A Vertex Claude deployment priced in model_info without a search price now bills each web search at the served model's cost map price, read at cost time and never written into the deployment entry. An explicit search price, zero included, still wins. * fix(vertex_ai): price web search on the Vertex Claude Sonnet 4.5 and Haiku 4.5 rows Google bills Claude web search at $10 per 1,000 searches on these models, so token-priced deployments of them no longer bill searches at 0 * fix(cost): bill served-model search price on token-only Anthropic and Azure AI Claude deployments Token-only Claude deployments on anthropic and azure_ai billed web search at 0 because the deployment entry has no search price. They now fall back to the served model's cost map search price, as vertex_ai already does. Vertex routing reads the served model first, so a Gemini deployment whose id contains claude keeps per-prompt grounding * fix(proxy): price streamed usage frames under the deployment's model so usage.cost matches the spend row * test(proxy): give the Vertex streamRawPredict logging doubles the request's model, as a real pass-through Logging has --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../llm_cost_calc/tool_call_cost_tracking.py | 3 + litellm/llms/__init__.py | 47 +++- ...odel_prices_and_context_window_backup.json | 20 ++ litellm/proxy/common_request_processing.py | 3 +- model_prices_and_context_window.json | 20 ++ .../cost_calculation/cases/vertex_ai.py | 237 +++++++++++++++++- .../test_tool_call_cost_tracking.py | 172 ++++++++++++- .../test_streaming_handler_interrupt.py | 5 +- .../test_success_handler.py | 2 + .../proxy/test_common_request_processing.py | 67 +++++ 10 files changed, 566 insertions(+), 10 deletions(-) diff --git a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py index 045869e4547..54213f40ac0 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py +++ b/litellm/litellm_core_utils/llm_cost_calc/tool_call_cost_tracking.py @@ -208,11 +208,14 @@ class StandardBuiltInToolCostTracking: usage=usage, response_object=response_object ) + response_model: Final = _output_item_field(response_object, "model") if model_info is not None and resolved_usage is not None and resolved_provider is not None: result: Final = get_cost_for_web_search_request( custom_llm_provider=resolved_provider, usage=resolved_usage, model_info=model_info, + response_model=response_model if isinstance(response_model, str) else None, + lookup_model_info=StandardBuiltInToolCostTracking._safe_get_model_info, ) if result is not None: return result diff --git a/litellm/llms/__init__.py b/litellm/llms/__init__.py index c18ce8cea68..84c91be9386 100644 --- a/litellm/llms/__init__.py +++ b/litellm/llms/__init__.py @@ -9,6 +9,8 @@ from litellm.types.utils import CallTypes from . import * if TYPE_CHECKING: + from collections.abc import Callable + from litellm.llms.base_llm.guardrail_translation.base_translation import ( BaseTranslation, ) @@ -30,7 +32,29 @@ def get_cost_for_google_maps_grounding_request( return cost_per_google_maps_grounding_request(usage=usage, model_info=model_info) -def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", model_info: "ModelInfo") -> float | None: +def _with_served_model_search_pricing( + model_info: "ModelInfo", + response_model: str | None, + served_model_provider: str, + lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None", +) -> "ModelInfo": + if model_info.get("search_context_cost_per_query") is not None or response_model is None: + return model_info + served_model_info: Final = ( + lookup_model_info(response_model, served_model_provider) if lookup_model_info is not None else None + ) + if served_model_info is None: + return model_info + return {**model_info, "search_context_cost_per_query": served_model_info.get("search_context_cost_per_query")} + + +def get_cost_for_web_search_request( + custom_llm_provider: str, + usage: "Usage", + model_info: "ModelInfo", + response_model: str | None = None, + lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None" = None, +) -> float | None: """ Get the cost for a web search request for a given model. @@ -38,6 +62,8 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo custom_llm_provider: The custom LLM provider. usage: The usage object. model_info: The model info. + response_model: The model the provider response names. + lookup_model_info: Looks up a model's cost map entry, returning None when it is not mapped. """ if custom_llm_provider == "gemini": from .gemini.cost_calculator import cost_per_web_search_request @@ -46,17 +72,25 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo elif custom_llm_provider == "anthropic": from .anthropic.cost_calculation import get_cost_for_anthropic_web_search - return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage) + return get_cost_for_anthropic_web_search( + model_info=_with_served_model_search_pricing(model_info, response_model, "anthropic", lookup_model_info), + usage=usage, + ) elif custom_llm_provider.startswith("vertex_ai"): # Anthropic Claude models on Vertex AI populate server_tool_use.web_search_requests # (same as the direct Anthropic API), not prompt_tokens_details.web_search_requests # (which is the Gemini field). Route claude-* models to the Anthropic calculator. model_key: Final[str] = model_info.get("key", "") if model_info else "" - if "claude" in model_key.lower(): + if "claude" in (response_model or model_key).lower(): from .anthropic.cost_calculation import get_cost_for_anthropic_web_search verbose_logger.debug("vertex_ai/claude model detected — routing web search cost to Anthropic calculator") - return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage) + return get_cost_for_anthropic_web_search( + model_info=_with_served_model_search_pricing( + model_info, response_model, "vertex_ai", lookup_model_info + ), + usage=usage, + ) from .vertex_ai.gemini.cost_calculator import ( cost_per_web_search_request as cost_per_web_search_request_vertex_ai, @@ -71,7 +105,10 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo return None from .anthropic.cost_calculation import get_cost_for_anthropic_web_search - return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage) + return get_cost_for_anthropic_web_search( + model_info=_with_served_model_search_pricing(model_info, response_model, "azure_ai", lookup_model_info), + usage=usage, + ) elif custom_llm_provider == "perplexity": # Perplexity handles search costs internally in its own cost calculator # Return 0.0 to indicate costs are already accounted for diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4d503952600..cbe502e5cae 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -51387,6 +51387,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -51415,6 +51420,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -52161,6 +52171,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -52266,6 +52281,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, "regional_endpoint_uplift_multiplier": 1.1, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index c9b6fff3a00..5599086d5c1 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -5031,8 +5031,9 @@ class ProxyBaseLLMRequestProcessing: if stream_usage is None: return None service_tier: Final = obj.get("service_tier") + served_model: Final = litellm_logging_obj.model if litellm_logging_obj is not None else None cost_val: Final = ProxyBaseLLMRequestProcessing._streamed_usage_cost( - ModelResponse(usage=stream_usage), + ModelResponse(model=served_model, usage=stream_usage), model_name, service_tier if isinstance(service_tier, str) else None, litellm_logging_obj, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4d503952600..cbe502e5cae 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -51387,6 +51387,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -51415,6 +51420,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 5e-06, "output_cost_per_token_batches": 2.5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -52161,6 +52171,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, "regional_endpoint_uplift_multiplier": 1.1, @@ -52266,6 +52281,11 @@ "max_input_tokens": 200000, "max_output_tokens": 64000, "mode": "chat", + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, "output_cost_per_token": 1.5e-05, "output_cost_per_token_batches": 7.5e-06, "regional_endpoint_uplift_multiplier": 1.1, diff --git a/tests/integration/cost_calculation/cases/vertex_ai.py b/tests/integration/cost_calculation/cases/vertex_ai.py index 2fb005dbeca..8590a19646b 100644 --- a/tests/integration/cost_calculation/cases/vertex_ai.py +++ b/tests/integration/cost_calculation/cases/vertex_ai.py @@ -3,7 +3,7 @@ PARITY pairs the streamed case with its plain twin where both bill the same row.""" import json -from typing import Final, cast +from typing import Final, Literal, cast from pydantic import JsonValue @@ -41,6 +41,7 @@ from integration.cost_calculation.cases.anthropic import ( from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs from integration.cost_calculation.cost_tracking_case import ( CostTrackingTestCase, + Deployment, ExactExpected, JsonResponse, RecountExpected, @@ -3387,6 +3388,232 @@ VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS: Final = bridge_pairs( ) +VERTEX_AI_CLAUDE_DEPLOYMENT_RATES: Final = Deployment( + input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.01 +) + + +def _vertex_claude_web_search_message(web_search_requests: int) -> JsonResponse: + return JsonResponse( + content_type="application/json", + body={ + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "scripted answer for cost fixtures"}], + "stop_reason": "end_turn", + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "server_tool_use": {"web_search_requests": web_search_requests}, + }, + }, + ) + + +def _vertex_claude_web_search_stream(web_search_requests: int) -> SseResponse: + return SseResponse( + content_type="text/event-stream", + frames=sse_frames( + { + "type": "message_start", + "message": { + "id": "msg_$REQUEST_ID", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [], + "stop_reason": None, + "usage": {"input_tokens": 1840}, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "scripted answer for cost fixtures"}, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"output_tokens": 412, "server_tool_use": {"web_search_requests": web_search_requests}}, + }, + {"type": "message_stop"}, + ), + ) + + +def _vertex_claude_deployment_web_search_case( + name: str, + endpoint: Literal["/v1/chat/completions", "/v1/responses", "/v1/messages"], + request: dict[str, JsonValue], + response: JsonResponse | SseResponse, + tool_usage_cost: float, + deployment: Deployment = VERTEX_AI_CLAUDE_DEPLOYMENT_RATES, +) -> CostTrackingTestCase: + return CostTrackingTestCase( + name=name, + covers="quota_management.spend_tracking.cost_matrix.logs_cost", + model="vertex_ai/claude-sonnet-5-5", + endpoint=endpoint, + deployment=deployment, + request={"model": "$MODEL", **request}, + response=response, + expected=ExactExpected( + spend=round(0.0039 + tool_usage_cost, 10), + input_cost=0.00184, + output_cost=0.00206, + prompt_tokens=1840, + completion_tokens=412, + tool_usage_cost=tool_usage_cost, + ), + ) + + +VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL: Final[dict[str, JsonValue]] = { + "type": "web_search_20250305", + "name": "web_search", + "max_uses": 3, +} + +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_rates_messages_web_search_two_requests", + "/v1/messages", + { + "max_tokens": 412, + "messages": [{"role": "user", "content": "vertex claude deployment rates messages web search"}], + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + }, + _vertex_claude_web_search_message(2), + 0.02, +) +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM: Final = _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_rates_messages_web_search_two_requests_stream", + "/v1/messages", + { + "max_tokens": 412, + "messages": [{"role": "user", "content": "vertex claude deployment rates messages web search stream"}], + "stream": True, + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + }, + _vertex_claude_web_search_stream(2), + 0.02, +) +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE: Final = ( + _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_zero_search_rate_messages_two_requests", + "/v1/messages", + { + "max_tokens": 412, + "messages": [{"role": "user", "content": "vertex claude deployment zero search rate messages"}], + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + }, + _vertex_claude_web_search_message(2), + 0.0, + Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.0), + ) +) +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY: Final = ( + _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_token_rates_only_messages_web_search_two_requests", + "/v1/messages", + { + "max_tokens": 412, + "messages": [{"role": "user", "content": "vertex claude deployment token rates only messages web search"}], + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + }, + _vertex_claude_web_search_message(2), + 0.034, + Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06), + ) +) +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_three_requests", + "/v1/chat/completions", + { + "messages": [{"role": "user", "content": "vertex claude deployment rates chat web search"}], + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + "allowed_openai_params": ["web_search_options"], + }, + _vertex_claude_web_search_message(3), + 0.03, +) +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM: Final = _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_two_requests_stream", + "/v1/chat/completions", + { + "messages": [{"role": "user", "content": "vertex claude deployment rates chat web search stream"}], + "stream": True, + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + "allowed_openai_params": ["web_search_options"], + }, + _vertex_claude_web_search_stream(2), + 0.02, +) +VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case( + "vertex_ai/claude-sonnet-5-5-deployment_rates_responses_web_search_two_requests", + "/v1/responses", + {"input": "vertex claude deployment rates responses web search", "tools": [{"type": "web_search"}]}, + _vertex_claude_web_search_message(2), + 0.02, +) + +GEMINI_3_1_PRO_PREVIEW_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM: Final = CostTrackingTestCase( + name="vertex_ai/gemini-3.1-pro-preview-deployment_rates_messages_web_search_two_queries_stream", + covers="quota_management.spend_tracking.scripted_wire.logs_cost", + model="vertex_ai/gemini-3.1-pro-preview", + endpoint="/v1/messages", + deployment=Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.03), + request={ + "model": "$MODEL", + "max_tokens": 412, + "messages": [{"role": "user", "content": "vertex gemini deployment rates messages web search stream"}], + "stream": True, + "tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL], + }, + response=SseResponse( + content_type="text/event-stream", + frames=sse_frames( + { + "candidates": [ + { + "content": {"parts": [{"text": "scripted answer for cost fixtures"}], "role": "model"}, + "index": 0, + } + ], + "modelVersion": "gemini-3.1-pro-preview", + }, + { + "candidates": [ + { + "content": {"parts": [{"text": ""}], "role": "model"}, + "finishReason": "STOP", + "index": 0, + "groundingMetadata": {"webSearchQueries": ["query 0", "query 1"]}, + } + ], + "usageMetadata": { + "promptTokenCount": 1840, + "candidatesTokenCount": 412, + "totalTokenCount": 2252, + "promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}], + }, + "modelVersion": "gemini-3.1-pro-preview", + }, + ), + ), + expected=ExactExpected( + spend=0.0339, + input_cost=0.00184, + output_cost=0.00206, + prompt_tokens=1840, + completion_tokens=412, + tool_usage_cost=0.03, + ), +) + GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT: Final = _gemini_case( GEMINI_3_8_FLASH_INPUT_TEXT, "vertex_ai/gemini-3.1-pro-preview-input_text", "vertex_ai/gemini-3.1-pro-preview", ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412), @@ -3725,6 +3952,14 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = ( VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT, VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_STREAM, *(pair.bridged for pair in VERTEX_AI_CLAUDE_SONNET_5_5_BRIDGE_PAIRS), + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH, + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM, + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE, + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY, + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH, + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM, + VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH, + GEMINI_3_1_PRO_PREVIEW_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM, GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT, GEMINI_3_1_PRO_PREVIEW_STREAM, GEMINI_3_1_PRO_PREVIEW_CACHE_READ, diff --git a/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py b/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py index e2452a928a4..9da2632943c 100644 --- a/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py +++ b/tests/unit/litellm_core_utils/llm_cost_calc/test_tool_call_cost_tracking.py @@ -1,3 +1,4 @@ +from collections.abc import Callable from typing import Final import pytest @@ -7,7 +8,7 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) from litellm.types.llms.openai import FileSearchTool, ResponsesAPIResponse, WebSearchOptions -from litellm.types.utils import ModelResponse, StandardBuiltInToolsParams +from litellm.types.utils import ModelResponse, ServerToolUse, StandardBuiltInToolsParams, Usage def test_web_search_cost_low(): @@ -1052,6 +1053,175 @@ def test_azure_ai_non_claude_web_search_keeps_generic_fallback() -> None: assert get_cost_for_web_search_request(custom_llm_provider="azure_ai", usage=usage, model_info=model_info) is None +def _register_priced_deployment( + model: str, search_cost_per_query: float | None, deployment_id: str | None = None +) -> str: + from litellm import Router + + token_rates: Final = {"input_cost_per_token": 3e-6, "output_cost_per_token": 1.5e-5} + search_rates: Final = ( + {} + if search_cost_per_query is None + else { + "search_context_cost_per_query": { + "search_context_size_low": search_cost_per_query, + "search_context_size_medium": search_cost_per_query, + "search_context_size_high": search_cost_per_query, + } + } + ) + router: Final = Router( + model_list=[ + { + "model_name": "priced-deployment", + "litellm_params": {"model": model, "api_key": "k", "vertex_project": "p", "vertex_location": "global"}, + "model_info": {**token_rates, **search_rates, **({} if deployment_id is None else {"id": deployment_id})}, + } + ] + ) + return router.model_list[0]["model_info"]["id"] + + +def _claude_chat_response( + web_search_requests: int, model: str = "claude-sonnet-5-5" +) -> tuple[object, Usage | None]: + usage: Final = Usage( + prompt_tokens=1840, + completion_tokens=412, + total_tokens=2252, + server_tool_use=ServerToolUse(web_search_requests=web_search_requests), + ) + return ModelResponse(model=model, usage=usage), usage + + +def _claude_messages_response( + web_search_requests: int, model: str = "claude-sonnet-5-5" +) -> tuple[object, Usage | None]: + return { + "id": "msg_vertex_claude_web_search", + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": "scripted answer"}], + "usage": { + "input_tokens": 1840, + "output_tokens": 412, + "server_tool_use": {"web_search_requests": web_search_requests}, + }, + }, None + + +@pytest.mark.parametrize("build_response", [_claude_chat_response, _claude_messages_response]) +@pytest.mark.parametrize(("web_search_requests", "search_cost_per_query"), [(2, 0.01), (3, 0.01), (2, 0.0)]) +def test_vertex_claude_deployment_priced_web_search_billed_per_request( + local_model_cost_map: None, + build_response: Callable[[int], tuple[object, Usage | None]], + web_search_requests: int, + search_cost_per_query: float, +) -> None: + deployment_id: Final = _register_priced_deployment("vertex_ai/claude-sonnet-5-5", search_cost_per_query) + response_object, usage = build_response(web_search_requests) + + cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=deployment_id, + response_object=response_object, + usage=usage, + custom_llm_provider="vertex_ai", + standard_built_in_tools_params=None, + ) + + assert cost == pytest.approx(web_search_requests * search_cost_per_query) + + +@pytest.mark.parametrize("build_response", [_claude_chat_response, _claude_messages_response]) +@pytest.mark.parametrize( + ("deployment_model", "served_model", "custom_llm_provider"), + # Google bills Claude web search on Sonnet 4.5 and Haiku 4.5 too: + # https://cloud.google.com/model-garden/partner-models/pricing (read 2026-10-30) + [ + ("vertex_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "vertex_ai"), + ("vertex_ai/claude-sonnet-4-5@20250929", "claude-sonnet-4-5-20250929", "vertex_ai"), + ("vertex_ai/claude-haiku-4-5@20251001", "claude-haiku-4-5-20251001", "vertex_ai"), + ("anthropic/claude-sonnet-5-5", "claude-sonnet-5-5", "anthropic"), + ("azure_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "azure_ai"), + ], +) +@pytest.mark.parametrize("deployment_id", [None, "claude-token-priced"], ids=["hashed_id", "claude_in_id"]) +def test_claude_deployment_with_token_rates_only_bills_served_model_search_price( + local_model_cost_map: None, + build_response: Callable[[int, str], tuple[object, Usage | None]], + deployment_model: str, + served_model: str, + custom_llm_provider: str, + deployment_id: str | None, +) -> None: + served_search_pricing: Final = ( + litellm.get_model_info(model=deployment_model).get("search_context_cost_per_query") or {} + ) + served_search_price: Final = served_search_pricing.get("search_context_size_medium") or 0.0 + registered_id: Final = _register_priced_deployment(deployment_model, None, deployment_id) + response_object, usage = build_response(2, served_model) + + cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=registered_id, + response_object=response_object, + usage=usage, + custom_llm_provider=custom_llm_provider, + standard_built_in_tools_params=None, + ) + + assert served_search_price > 0 + assert cost == pytest.approx(2 * served_search_price) + + +def test_vertex_claude_deployment_with_token_rates_only_and_unmapped_served_model_bills_no_search( + local_model_cost_map: None, +) -> None: + deployment_id: Final = _register_priced_deployment("vertex_ai/claude-sonnet-5-5", None) + usage: Final = Usage( + prompt_tokens=1840, + completion_tokens=412, + total_tokens=2252, + server_tool_use=ServerToolUse(web_search_requests=2), + ) + + cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=deployment_id, + response_object=ModelResponse(model="claude-unmapped-preview", usage=usage), + usage=usage, + custom_llm_provider="vertex_ai", + standard_built_in_tools_params=None, + ) + + assert cost == 0.0 + + +@pytest.mark.parametrize("deployment_id", [None, "claude-code-gemini"], ids=["hashed_id", "claude_in_id"]) +def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt( + local_model_cost_map: None, deployment_id: str | None +) -> None: + search_cost_per_prompt: Final = 0.03 + registered_id: Final = _register_priced_deployment( + "vertex_ai/gemini-2.5-flash", search_cost_per_prompt, deployment_id + ) + usage: Final = Usage( + prompt_tokens=1840, + completion_tokens=412, + total_tokens=2252, + server_tool_use=ServerToolUse(web_search_requests=2), + ) + + cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools( + model=registered_id, + response_object=ModelResponse(model="gemini-2.5-flash", usage=usage), + usage=usage, + custom_llm_provider="vertex_ai", + standard_built_in_tools_params=None, + ) + + assert cost == pytest.approx(search_cost_per_prompt) + + @pytest.mark.parametrize( "kwargs", [{"tools": 5}, {"web_search_options": None}, {"web_search_options": 5}], diff --git a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py index ae95c27950c..8127ffe558b 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py +++ b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py @@ -41,12 +41,13 @@ def _make_streaming_response(chunks): return mock -def _unarmed_logging_obj(): +def _unarmed_logging_obj(model: str = "unknown"): """Real Logging objects only carry _on_deferred_stream_complete when the proxy arms deferred dispatch; a bare MagicMock's auto-attribute is truthy and would spuriously trigger the deferral branch.""" obj = MagicMock() obj._on_deferred_stream_complete = None + obj.model = model return obj @@ -409,7 +410,7 @@ async def _collect_openai_passthrough_chunks(chunks, endpoint_type): async for chunk in PassThroughStreamingHandler.chunk_processor( response=response, request_body={"model": "gpt-4o-mini", "stream": True}, - litellm_logging_obj=_unarmed_logging_obj(), + litellm_logging_obj=_unarmed_logging_obj(model="gpt-4o-mini"), endpoint_type=endpoint_type, start_time=datetime(2026, 1, 1), passthrough_success_handler_obj=MagicMock(), diff --git a/tests/unit/proxy/pass_through_endpoints/test_success_handler.py b/tests/unit/proxy/pass_through_endpoints/test_success_handler.py index c721e808ed5..cf00545f331 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_success_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/test_success_handler.py @@ -346,6 +346,7 @@ async def test_vertex_ai_anthropic_streaming_cost_injection_enabled(): # Setup logging object with model info litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj) + litellm_logging_obj.model = "claude-sonnet-4@20250514" litellm_logging_obj.litellm_params = {} litellm_logging_obj.model_call_details = {"model": "claude-sonnet-4@20250514"} litellm_logging_obj.completion_start_time = None @@ -544,6 +545,7 @@ async def test_vertex_ai_anthropic_streaming_model_extraction(): response.aiter_bytes = mock_aiter_bytes litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj) + litellm_logging_obj.model = "claude-sonnet-4@20250514" litellm_logging_obj.litellm_params = {} litellm_logging_obj.model_call_details = {} litellm_logging_obj.completion_start_time = None diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index fe6b894b368..4d1b67f9212 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -74,6 +74,7 @@ from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitErro from litellm.proxy.common_utils.sse_keepalive import ANTHROPIC_PING_SSE_CHUNK from litellm.proxy.dd_span_tagger import DDSpanTagger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.proxy._types import Litellm_EntityType, ProxyErrorTypes, ProxyException from litellm.proxy._types import UserAPIKeyAuth as ProxyUserAPIKeyAuth from litellm.proxy.auth.fallback_budget import RouterFallbackBudgetCheck @@ -10221,6 +10222,8 @@ class TestInjectCostIntoUsageDict: discount streamed a ``usage.cost`` that disagreed with the callback's ``response_cost``.""" class _StubLoggingObj: + model = "claude-haiku-4-5" + def __init__(self, cost): self._cost = cost self.captured_result = None @@ -10257,6 +10260,8 @@ class TestInjectCostIntoUsageDict: def test_message_delta_falls_back_to_model_pricing_when_the_logging_obj_returns_no_cost(self): class _StubLoggingObj: + model = "claude-haiku-4-5" + def response_cost_calculator(self, result): return None @@ -10282,6 +10287,8 @@ class TestInjectCostIntoUsageDict: model-name pricing rather than propagating into the response body.""" class _StubLoggingObj: + model = "claude-haiku-4-5" + def response_cost_calculator(self, result): raise ValueError("no pricing for this deployment") @@ -10363,6 +10370,8 @@ class TestInjectCostIntoUsageDict: streaming /v1/chat/completions gets its negotiated price instead of sticker.""" class _StubLoggingObj: + model = "gpt-4o-mini" + def __init__(self, cost): self._cost = cost self.captured_result = None @@ -10391,6 +10400,8 @@ class TestInjectCostIntoUsageDict: def test_openai_chunk_falls_back_to_model_pricing_when_the_logging_obj_returns_no_cost(self): class _StubLoggingObj: + model = "gpt-4o-mini" + def response_cost_calculator(self, result): return None @@ -10406,6 +10417,60 @@ class TestInjectCostIntoUsageDict: assert result is not None assert result["usage"]["cost"] == pytest.approx(self._expected_cost("gpt-4o-mini", 11, 4)) + @pytest.mark.parametrize( + ("deployment_model", "served_model", "custom_llm_provider"), + [ + ("vertex_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "vertex_ai"), + ("anthropic/claude-sonnet-5-5", "claude-sonnet-5-5", "anthropic"), + ("azure_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "azure_ai"), + ], + ) + def test_message_delta_bills_search_like_the_logged_response_on_a_token_priced_claude_deployment( + self, local_model_cost_map: None, deployment_model: str, served_model: str, custom_llm_provider: str + ) -> None: + token_rates: Final = {"input_cost_per_token": 3e-6, "output_cost_per_token": 1.5e-5} + router: Final = Router( + model_list=[ + { + "model_name": "priced-claude", + "litellm_params": { + "model": deployment_model, + "api_key": "k", + "vertex_project": "p", + "vertex_location": "global", + }, + "model_info": token_rates, + } + ] + ) + deployment_id: Final = router.model_list[0]["model_info"]["id"] + logging_obj: Final = LiteLLMLoggingObj( + model=deployment_model, + messages=[{"role": "user", "content": "test"}], + stream=True, + call_type="anthropic_messages", + start_time=None, + litellm_call_id="token-priced-claude-search", + function_id="token-priced-claude-search", + ) + logging_obj.update_environment_variables( + litellm_params={"metadata": {"model_info": {"id": deployment_id, **token_rates}}}, optional_params={} + ) + logging_obj.model_call_details["custom_llm_provider"] = custom_llm_provider + wire_usage: Final = {"input_tokens": 1840, "output_tokens": 412, "server_tool_use": {"web_search_requests": 2}} + event: Final = {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": wire_usage} + + result: Final = ProxyBaseLLMRequestProcessing._inject_cost_into_usage_dict(event, "priced-claude", logging_obj) + + stream_usage: Final = AnthropicConfig().calculate_usage(usage_object=wire_usage, reasoning_content=None) + logged_cost: Final = logging_obj.response_cost_calculator( + result=litellm.ModelResponse(model=served_model, usage=stream_usage) + ) + token_cost: Final = 1840 * token_rates["input_cost_per_token"] + 412 * token_rates["output_cost_per_token"] + assert logged_cost is not None and logged_cost > token_cost + assert result is not None + assert result["usage"]["cost"] == pytest.approx(logged_cost) + class TestProcessChunkWithCostInjection: @pytest.mark.parametrize("model", ["claude-opus-5-5", "vertex_ai/claude-opus-5-5"]) @@ -10472,6 +10537,8 @@ class TestProcessChunkWithCostInjection: monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True) class _StubLoggingObj: + model = "claude-haiku-4-5" + def response_cost_calculator(self, result): return 0.00042 From af070e7cd0152acb704c1965ffb5f67ae80eee57 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:12:10 -0700 Subject: [PATCH 4/6] build(deps): bump pyasn1, pypdf and minimatch to clear osv-scan (#49362) * build(deps): bump pyasn1 from 0.6.4 to 0.6.5 Bumps [pyasn1](https://github.com/pyasn1/pyasn1) from 0.6.4 to 0.6.5. - [Release notes](https://github.com/pyasn1/pyasn1/releases) - [Changelog](https://github.com/pyasn1/pyasn1/blob/main/CHANGES.rst) - [Commits](https://github.com/pyasn1/pyasn1/compare/v0.6.4...v0.6.5) --- updated-dependencies: - dependency-name: pyasn1 dependency-version: 0.6.5 dependency-type: indirect ... Signed-off-by: dependabot[bot] * build(deps): bump pyasn1, pypdf and minimatch to clear osv-scan pyasn1 0.6.4 -> 0.6.5 and pypdf 6.21.2 -> 6.21.4 in uv.lock, with a dated exclude-newer-package cutoff that admits pypdf 6.21.4 before it clears the 3-day window. minimatch has no fixed 10.x release, so the dashboard's override and a new vscode-extension override move it to 11.0.2. --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- pyproject.toml | 2 ++ ui/litellm-dashboard/package-lock.json | 16 ++++++++++++---- ui/litellm-dashboard/package.json | 2 +- uv.lock | 13 +++++++------ vscode-extension/package-lock.json | 16 ++++++++++++---- vscode-extension/package.json | 3 +++ 6 files changed, 37 insertions(+), 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6e9e75a5b7f..5c23a64ebb3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -383,6 +383,8 @@ exclude-newer = "3 days" [tool.uv.exclude-newer-package] # TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window brotli = "2026-10-22T14:01:00Z" +# TODO: remove after 2026-10-31, once pypdf 6.21.4 clears the 3-day window +pypdf = "2026-10-28T17:01:00Z" [tool.uv.sources] litellm-proxy-extras = { workspace = true } diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 48def46e3fe..54cd9087c4e 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -2037,6 +2037,13 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@isaacs/nonrecursive-glob-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.2.tgz", + "integrity": "sha512-GJPHIgPPtFl+nLhFHyF0zZoFcOFLsZWbHFFzo+Zr+PWDXJbl9hHxb2w+9SiF2fHmLgKaXIpUzQo3SbBBZ9WObw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, "node_modules/@jridgewell/gen-mapping": { "version": "0.3.13", "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", @@ -9657,13 +9664,14 @@ } }, "node_modules/minimatch": { - "version": "10.2.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz", - "integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.2.tgz", + "integrity": "sha512-pWTgz5CN3tjqyZNSK2tePlSmu9IwM41q8yn8Usdqqwe9TIHh+9dEpLLjF4uApsfPEAZY71qtSiA2B2APEr+1Yw==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.2" + "@isaacs/nonrecursive-glob-match": "^1.0.2", + "brace-expansion": "^5.0.12" }, "engines": { "node": "18 || 20 || >=22" diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index f65316439bf..0707496a6bc 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -117,7 +117,7 @@ "js-yaml": "4.3.2", "brace-expansion": "5.0.12", "glob": "13.0.0", - "minimatch": "10.2.4", + "minimatch": "11.0.2", "ws": "8.21.0", "braces": "3.0.3", "axios": "1.13.6", diff --git a/uv.lock b/uv.lock index 8cd31eb0828..19d3a1ffd4f 100644 --- a/uv.lock +++ b/uv.lock @@ -16,6 +16,7 @@ exclude-newer-span = "P3D" [options.exclude-newer-package] brotli = "2026-10-22T14:01:00Z" +pypdf = "2026-10-28T17:01:00Z" [manifest] members = [ @@ -8372,11 +8373,11 @@ wheels = [ [[package]] name = "pyasn1" -version = "0.6.4" +version = "0.6.5" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" } +sdist = { url = "https://files.pythonhosted.org/packages/88/7e/3c0da41061a2021cac7cd524fd57092adbe5b515e8bf9005702e366802e2/pyasn1-0.6.5.tar.gz", hash = "sha256:6b11616a6bcf3a40e635f082bb232fa57b5e0dd7f517bd7ddd69ec3794508a47", size = 155152, upload-time = "2026-10-27T06:32:49.125Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" }, + { url = "https://files.pythonhosted.org/packages/19/1f/3eefd5c84c35cae44549ce1c7f7e58f7945b34f5a6bce07a135438522073/pyasn1-0.6.5-py3-none-any.whl", hash = "sha256:4017b0fce125096b8573b39c989fdb1a4b19cf761e683398184550fea0e66587", size = 84807, upload-time = "2026-10-27T06:32:48.045Z" }, ] [[package]] @@ -8869,14 +8870,14 @@ wheels = [ [[package]] name = "pypdf" -version = "6.21.2" +version = "6.21.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/06/28/081f1cbcca7042441dafc8de00f48ba60853f89e5bff92c5f26477346d81/pypdf-6.21.2.tar.gz", hash = "sha256:0a3e1c5c886a1be03646c16a04827cc166c2a0575c971b26a4bf89686551c2d4", size = 7144571, upload-time = "2026-10-21T16:44:55.272Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1e/e1/ec8f4d155c997c45917153a5521e60d6ee6389d68abdcbc8e44e564caffc/pypdf-6.21.4.tar.gz", hash = "sha256:c8817e3f888d47a087872ded69812c1ef30cacb871d6e053e0c00bbaf0bc2163", size = 7109255, upload-time = "2026-10-28T17:00:54.81Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/4d/c9d2d67f204eff463237a1a8f5e25a6923b7b1043982b91fcaf104c8ef60/pypdf-6.21.2-py3-none-any.whl", hash = "sha256:1d44d1a3a200f25062adff0085b02040e4646431ac2ccac22e42679e91466084", size = 407766, upload-time = "2026-10-21T16:44:53.626Z" }, + { url = "https://files.pythonhosted.org/packages/3f/4b/77e8ad4ac42c9d93600f40541a50e9feb3a61d2bbee2d76f659346072051/pypdf-6.21.4-py3-none-any.whl", hash = "sha256:33e4fb41ca352810eb4e9838aa0d82d6888a5f8bf682f43e99cce5faa666255c", size = 410430, upload-time = "2026-10-28T17:00:53.146Z" }, ] [[package]] diff --git a/vscode-extension/package-lock.json b/vscode-extension/package-lock.json index 5f6cf7edf4e..80b868e528b 100644 --- a/vscode-extension/package-lock.json +++ b/vscode-extension/package-lock.json @@ -644,6 +644,13 @@ "node": ">=18" } }, + "node_modules/@isaacs/nonrecursive-glob-match": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.2.tgz", + "integrity": "sha512-GJPHIgPPtFl+nLhFHyF0zZoFcOFLsZWbHFFzo+Zr+PWDXJbl9hHxb2w+9SiF2fHmLgKaXIpUzQo3SbBBZ9WObw==", + "dev": true, + "license": "BlueOak-1.0.0" + }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", @@ -2700,13 +2707,14 @@ } }, "node_modules/minimatch": { - "version": "10.2.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", - "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "version": "11.0.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.2.tgz", + "integrity": "sha512-pWTgz5CN3tjqyZNSK2tePlSmu9IwM41q8yn8Usdqqwe9TIHh+9dEpLLjF4uApsfPEAZY71qtSiA2B2APEr+1Yw==", "dev": true, "license": "BlueOak-1.0.0", "dependencies": { - "brace-expansion": "^5.0.8" + "@isaacs/nonrecursive-glob-match": "^1.0.2", + "brace-expansion": "^5.0.12" }, "engines": { "node": "18 || 20 || >=22" diff --git a/vscode-extension/package.json b/vscode-extension/package.json index a3cf2dc7a5f..42626439aca 100644 --- a/vscode-extension/package.json +++ b/vscode-extension/package.json @@ -83,5 +83,8 @@ "esbuild": "^0.28.2", "typescript": "^5.9.3", "vitest": "^4.1.12" + }, + "overrides": { + "minimatch": "11.0.2" } } From e5aa3d32cde109d236881ff5473d3d43ff0a384e Mon Sep 17 00:00:00 2001 From: mika-lt <349045450+mika-lt@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:16:42 -0700 Subject: [PATCH 5/6] fix(proxy): forward a valid metadata.user_id to Anthropic Messages on pass-through routes (#49333) * fix(proxy): forward the client's own metadata on Anthropic and custom pass-through routes * fix(proxy): forward only a valid metadata.user_id to Anthropic Messages on pass-through routes The whole-object forward sent keys Anthropic refuses and reached other providers, so a request that used to pass could start failing with a 400. Read the client's user_id after hooks, drop it unless it is a string with no email or phone number, and send only that key when the upstream URL path ends in /v1/messages. Share the validator with the translated route, whose email check now matches any embedded address as Anthropic's does. * fix(proxy): forward metadata.user_id only to Anthropic upstreams no guardrail scans Limit the pass-through user_id to Messages create on api.anthropic.com or the configured Anthropic base, mirror the checks Anthropic answers 400 for, and keep the field off any request a pre-call guardrail inspects. The translated route goes back to its own user_id filter. * fix(proxy): judge user_id forwarding on the whole request and cap it for configured Anthropic bases * refactor(proxy): pass the user_id check a plain mapping instead of an optional body * fix(proxy): drop an unprintable metadata.user_id and warn when a refused id is dropped Anthropic's whitespace is not Python's, so a control character before the at sign hid an email from the filter and turned a 200 into a 400. Fireworks and Moonshot also fail on newline ids. Requiring a printable id covers both, and a refused id now logs a warning that never carries the value. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/constants.py | 2 + litellm/llms/anthropic/common_utils.py | 37 +++ .../llm_passthrough_endpoints.py | 8 +- .../pass_through_endpoints.py | 54 +++- litellm/proxy/utils.py | 13 +- .../test_anthropic_passthrough_wire.py | 1 - .../anthropic/test_anthropic_common_utils.py | 92 ++++++ .../test_pass_through_endpoints.py | 268 +++++++++++++++++- .../utils/proxy_logging/test_pre_call_hook.py | 18 ++ 9 files changed, 479 insertions(+), 14 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index c17e776f54b..28e7b4a7fd7 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -634,6 +634,8 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60 BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024) # Anthropic's Messages API rejects thinking.budget_tokens < 1024. ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024 +ANTHROPIC_MAX_USER_ID_LENGTH: Final = 512 +ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH: Final = 255 ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:" REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5) DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000) diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index e1f9a7d540f..e0da8943a4a 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -4,6 +4,7 @@ This file contains common utils for anthropic calls. import copy import json +import os import re from collections.abc import Container, Mapping, MutableMapping, Sequence from dataclasses import dataclass @@ -19,6 +20,8 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm.constants import ( + ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, + ANTHROPIC_MAX_USER_ID_LENGTH, DEFAULT_MODEL_CREATED_AT_TIME, DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET, DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET, @@ -178,6 +181,40 @@ def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: boo return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools] +def anthropic_pass_through_api_base() -> str: + return os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com" + + +def _same_host_and_port(url: httpx.URL, other: str) -> bool: + try: + other_url: Final = httpx.URL(other) + except httpx.InvalidURL: + return False + return (url.host, url.port) == (other_url.host, other_url.port) + + +def anthropic_upstream_user_id_limit(url: str) -> int | None: + upstream: Final = httpx.URL(url) + if not upstream.path.removesuffix("/").endswith("/v1/messages"): + return None + if upstream.host == "api.anthropic.com": + return ANTHROPIC_MAX_USER_ID_LENGTH + return ( + ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH + if _same_host_and_port(upstream, anthropic_pass_through_api_base()) + else None + ) + + +def _has_email_shape(token: str) -> bool: + at_sign: Final = token.find("@", 1) + return at_sign != -1 and token.find(".", at_sign + 2, -1) != -1 + + +def anthropic_accepts_user_id(user_id: str, max_length: int) -> bool: + return len(user_id) <= max_length and user_id.isprintable() and not any(map(_has_email_shape, user_id.split())) + + def is_anthropic_messages_url(url: str) -> bool: """Check whether a URL addresses Anthropic's Messages API.""" parsed_url: Final = urlparse(url) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 33b18c3d939..151938d73d7 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -52,7 +52,11 @@ from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization -from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers +from litellm.llms.anthropic.common_utils import ( + AnthropicModelInfo, + anthropic_pass_through_api_base, + merge_anthropic_beta_headers, +) from litellm.llms.azure.passthrough.transformation import ( foreign_azure_deployment, is_azure_body_model_inference_endpoint, @@ -1047,7 +1051,7 @@ async def anthropic_proxy_route( """ [Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion) """ - base_target_url = os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com" + base_target_url = anthropic_pass_through_api_base() encoded_endpoint = httpx.URL(endpoint).path # Ensure endpoint starts with '/' for proper URL construction diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 647e145aac1..bda4f4c2ad2 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -71,7 +71,11 @@ from litellm.litellm_core_utils.litellm_logging import get_masked_values from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.litellm_core_utils.redact_messages import should_redact_message_logging from litellm.litellm_core_utils.safe_json_dumps import safe_dumps -from litellm.llms.anthropic.common_utils import is_anthropic_messages_url +from litellm.llms.anthropic.common_utils import ( + anthropic_accepts_user_id, + anthropic_upstream_user_id_limit, + is_anthropic_messages_url, +) from litellm.llms.base_llm.managed_resources.utils import ( resolve_passthrough_managed_id_provider, ) @@ -503,7 +507,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): url: httpx.URL, headers: dict, requested_query_params: dict | None = None, - _parsed_body: dict | None = None, + _parsed_body: Mapping[str, object] | None = None, forward_multipart: bool = False, ) -> httpx.Response: """ @@ -717,8 +721,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): # unenforced. Set after the client merge so a request body cannot supply its own budget. # # Only for the built-in provider routes. `get_model_from_request` returns - # None for a user-defined pass-through, deliberately: its body is forwarded - # verbatim, so `model` there names an UPSTREAM model rather than a + # None for a user-defined pass-through, deliberately: the client's `model` + # goes upstream unchanged, so it names an UPSTREAM model rather than a # LiteLLM-managed one. Enforcement is therefore skipped on those routes, and # charging a counter anyway would track spend that nothing can refuse, and # would attribute it to a budget the operator scoped to a LiteLLM model that @@ -1187,6 +1191,36 @@ def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]: return headers +def _json_object(value: object) -> Mapping[str, object]: + return cast("Mapping[str, object]", value) if isinstance(value, Mapping) else MappingProxyType({}) # cast-ok: json + + +def _anthropic_user_id_to_forward( + body: Mapping[str, object], + url: str, + has_pre_call_guardrails: Callable[[Mapping[str, object]], bool], +) -> str | None: + user_id: Final = _json_object(body.get("metadata")).get("user_id") + if not isinstance(user_id, str): + return None + max_length: Final = anthropic_upstream_user_id_limit(url) + if max_length is None: + return None + if not anthropic_accepts_user_id(user_id, max_length): + verbose_proxy_logger.warning("Not forwarding metadata.user_id because the upstream would refuse it") + return None + if has_pre_call_guardrails(body): + verbose_proxy_logger.debug("Not forwarding metadata.user_id because a pre-call guardrail cannot scan it") + return None + return user_id + + +def _with_anthropic_user_id(body: Mapping[str, object] | None, user_id: str | None) -> Mapping[str, object] | None: + if body is None or user_id is None: + return body + return {**body, "metadata": {"user_id": user_id}} + + def _eu_jev_client_body(custom_llm_provider: str | None, body: Mapping[str, object]) -> Mapping[str, object] | None: return MappingProxyType(dict(body)) if custom_llm_provider == "eu_jev" else None @@ -1423,6 +1457,10 @@ async def pass_through_request( request_method=getattr(request, "method", None), cost_per_request=cost_per_request, ) + hooked_body: Final = _json_object(_parsed_body) + client_user_id: Final = _anthropic_user_id_to_forward( + hooked_body, str(url), proxy_logging_obj.request_has_pre_call_guardrails + ) kwargs = HttpPassThroughEndpointHelpers.init_kwargs_for_pass_through_endpoint( # rebind-ok: pre-existing rebinding on a rename-only line user_api_key_dict=user_api_key_dict, _parsed_body=_parsed_body, @@ -1590,11 +1628,13 @@ async def pass_through_request( else: logging_url = str(url) + "?" + requested_query_params_str + outgoing_body: Final = cast("Mapping[str, object] | None", _parsed_body) # cast-ok: json + upstream_body: Final = _with_anthropic_user_id(outgoing_body, client_user_id) logging_obj.pre_call( input=[{"role": "user", "content": safe_dumps(_parsed_body)}], api_key="", additional_args={ - "complete_input_dict": _parsed_body, + "complete_input_dict": upstream_body, "api_base": str(logging_url), "headers": redact_credential_headers(upstream_headers), }, @@ -1634,7 +1674,7 @@ async def pass_through_request( url, params=requested_query_params, headers=upstream_headers, - json=_parsed_body, + json=upstream_body, ) ) @@ -1733,7 +1773,7 @@ async def pass_through_request( url=url, headers=upstream_headers, requested_query_params=requested_query_params, - _parsed_body=_parsed_body, + _parsed_body=upstream_body, forward_multipart=is_multipart, ) verbose_proxy_logger.debug("response.headers= %s", response.headers) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 9e237ffcb57..c7efbf9dacf 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3031,14 +3031,21 @@ class ProxyLogging: hook to consult, but it judges the payload the same way, so a proxy configured only with one of those still has something to say about every record. """ - if request_metadata.get("_guardrail_pipelines"): + return self.request_has_pre_call_guardrails({"metadata": dict(request_metadata)}, event_type) + + def request_has_pre_call_guardrails( + self, + request_data: Mapping[str, object], + event_type: GuardrailEventHooks = GuardrailEventHooks.pre_call, + ) -> bool: + if _policy_state_metadata(request_data).get("_guardrail_pipelines"): return True caps: Final = ProxyLogging.callback_capabilities() if caps.has_content_enforcer: return True - probe: Final = {"metadata": dict(request_metadata)} return any( - isinstance(callback, CustomGuardrail) and callback.should_run_guardrail(data=probe, event_type=event_type) + isinstance(callback, CustomGuardrail) + and callback.should_run_guardrail(data=request_data, event_type=event_type) for callback in caps.resolved_callbacks ) diff --git a/tests/integration/providers/test_anthropic_passthrough_wire.py b/tests/integration/providers/test_anthropic_passthrough_wire.py index a791eb9c2a2..0a16520f626 100644 --- a/tests/integration/providers/test_anthropic_passthrough_wire.py +++ b/tests/integration/providers/test_anthropic_passthrough_wire.py @@ -317,7 +317,6 @@ def test_credentialless_proxy_forwards_the_caller_anthropic_credential_and_never def test_anthropic_sdk_metadata_user_id_reaches_upstream(api_key_proxy: Gateway, wire: Wire) -> None: - pytest.skip("BUG: /anthropic/v1/messages drops the native metadata.user_id field before forwarding upstream") with api_key_proxy.scenario() as scenario: key: Final = scenario.key() sent: Final[list[httpx.Request]] = [] diff --git a/tests/unit/llms/anthropic/test_anthropic_common_utils.py b/tests/unit/llms/anthropic/test_anthropic_common_utils.py index d18ef28c68e..cf0b8a39865 100644 --- a/tests/unit/llms/anthropic/test_anthropic_common_utils.py +++ b/tests/unit/llms/anthropic/test_anthropic_common_utils.py @@ -56,6 +56,98 @@ def test_is_anthropic_messages_url(url: str, expected: bool) -> None: assert is_anthropic_messages_url(url) is expected +@pytest.mark.parametrize( + ("configured_base", "url", "expected"), + [ + pytest.param(None, "https://api.anthropic.com/v1/messages", 512, id="default-upstream"), + pytest.param(None, "https://api.anthropic.com:443/v1/messages/", 512, id="default-upstream-explicit-port"), + pytest.param("http://relay.test:8080", "https://api.anthropic.com/v1/messages", 512, id="direct-target"), + pytest.param("https://api.anthropic.com", "https://api.anthropic.com/v1/messages", 512, id="configured-default"), + pytest.param("http://relay.test:8080", "http://relay.test:8080/v1/messages", 255, id="configured-upstream"), + pytest.param("http://relay.test:8080/a", "http://relay.test:8080/a/v1/messages", 255, id="configured-prefix"), + pytest.param("https://Relay.Test", "https://relay.test/v1/messages", 255, id="configured-upper-case-host"), + pytest.param("https://relay.test:443", "https://relay.test/v1/messages", 255, id="configured-https-port"), + pytest.param("http://relay.test:80/a", "http://relay.test/a/v1/messages", 255, id="configured-http-port"), + pytest.param("http://relay.test:8080", "http://relay.test:8081/v1/messages", None, id="other-port"), + pytest.param("http://relay.test:notaport", "http://relay.test/v1/messages", None, id="invalid-configured-base"), + pytest.param(None, "https://other-vendor.test/v1/messages", None, id="other-vendor"), + pytest.param(None, "https://api.anthropic.com/v1/messages/count_tokens", None, id="count-tokens"), + ], +) +def test_anthropic_upstream_user_id_limit( + monkeypatch: pytest.MonkeyPatch, configured_base: str | None, url: str, expected: int | None +) -> None: + from litellm.llms.anthropic.common_utils import anthropic_upstream_user_id_limit + + monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False) + if configured_base is not None: + monkeypatch.setenv("ANTHROPIC_API_BASE", configured_base) + + assert anthropic_upstream_user_id_limit(url) == expected + + +def test_anthropic_pass_through_api_base_prefers_the_api_base_variable(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.llms.anthropic.common_utils import anthropic_pass_through_api_base + + monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://fallback.test") + monkeypatch.setenv("ANTHROPIC_API_BASE", "http://preferred.test") + assert anthropic_pass_through_api_base() == "http://preferred.test" + + monkeypatch.delenv("ANTHROPIC_API_BASE") + assert anthropic_pass_through_api_base() == "http://fallback.test" + + +# Each row was sent to https://api.anthropic.com/v1/messages on 2026-10-30, False rows answered 400. +@pytest.mark.parametrize( + ("user_id", "expected"), + [ + pytest.param("user-1", True, id="opaque"), + pytest.param("", True, id="empty"), + pytest.param('{"device_id":"abc","session_id":"def"}', True, id="claude-code-shape"), + pytest.param("1234567890", True, id="digits"), + pytest.param("+14155550100", True, id="phone-shape"), + pytest.param("a@b", True, id="no-dot"), + pytest.param("@b.co", True, id="nothing-before-at-sign"), + pytest.param("a@.co", True, id="nothing-between-at-sign-and-dot"), + pytest.param("a@b.", True, id="nothing-after-dot"), + pytest.param("a @ b.co", True, id="spaces-around-at-sign"), + pytest.param("a\u00a0@b.co", False, id="no-break-space-before-at-sign"), + pytest.param("a@b\n.co", False, id="newline-before-dot"), + pytest.param("line one\nline two", False, id="newline"), + pytest.param("a\tb", False, id="tab"), + pytest.param("a\x1f@b.co", False, id="unit-separator-before-at-sign"), + pytest.param("a@\x1cb.co", False, id="file-separator-after-at-sign"), + pytest.param("a\u200bb", False, id="zero-width-space"), + pytest.param("a@b.co", False, id="email"), + pytest.param("user a@b.co here", False, id="email-inside-text"), + pytest.param('{"device_id":"a@b.co"}', False, id="email-inside-json"), + pytest.param("a@@example.com", False, id="doubled-at-sign"), + pytest.param("@@example.com", False, id="leading-doubled-at-sign"), + pytest.param("a@b@c.co", False, id="two-at-signs"), + pytest.param("a@b..co", False, id="doubled-dot"), + pytest.param("a@.b.co", False, id="dot-after-at-sign"), + pytest.param("a@b.c d", False, id="email-before-space"), + ], +) +def test_anthropic_accepts_user_id(user_id: str, expected: bool) -> None: + from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH + from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id + + assert anthropic_accepts_user_id(user_id, ANTHROPIC_MAX_USER_ID_LENGTH) is expected + + +@pytest.mark.parametrize("character", ["a", "\u00e9", "\U0001f600"], ids=["ascii", "two-bytes", "four-bytes"]) +def test_anthropic_accepts_user_id_counts_characters_up_to_the_limit(character: str) -> None: + from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH + from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id + + assert anthropic_accepts_user_id(character * ANTHROPIC_MAX_USER_ID_LENGTH, ANTHROPIC_MAX_USER_ID_LENGTH) is True + assert ( + anthropic_accepts_user_id(character * (ANTHROPIC_MAX_USER_ID_LENGTH + 1), ANTHROPIC_MAX_USER_ID_LENGTH) is False + ) + + def test_anthropic_model_not_found_maps_to_not_found_but_authentication_error_does_not() -> None: with respx.mock(assert_all_called=True) as router: router.post("https://api.anthropic.com/v1/messages").mock( diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index b7e76ce0068..4fe358c38a3 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -31,7 +31,7 @@ from starlette.datastructures import UploadFile as StarletteUploadFile import litellm from litellm._logging import verbose_proxy_logger from litellm.caching.dual_cache import DualCache -from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS +from litellm.constants import ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, ANTHROPIC_MAX_USER_ID_LENGTH, DEFAULT_REQUEST_TIMEOUT_SECONDS from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.prometheus import PrometheusLogger @@ -6621,6 +6621,272 @@ async def test_pass_through_body_stays_byte_identical_outside_the_opt_in( assert "x-litellm-call-id" in response.headers +def _echo_upstream_body(upstream_request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, content=upstream_request.content, headers={"content-type": "application/json"}, request=upstream_request + ) + + +async def _relayed_body(response: Response) -> bytes: + if isinstance(response, StreamingResponse): + return b"".join([chunk async for chunk in response.body_iterator]) + return bytes(response.body) + + +async def _relay_through_pass_through( + client_body: Mapping[str, object], + target: str, + guardrails_config: dict[str, object] | None, + custom_body: dict[str, object] | None = None, +) -> Mapping[str, object]: + async def receive() -> Mapping[str, object]: + return {"type": "http.request", "body": json.dumps(dict(client_body)).encode()} + + request: Final = Request( + { + "type": "http", + "method": "POST", + "path": "/my-claude-route", + "query_string": b"", + "headers": [(b"content-type", b"application/json")], + }, + receive, + ) + fake_client, cleanup = _inject_fake_passthrough_client(httpx.MockTransport(_echo_upstream_body), timeout=None) + try: + with ExitStack() as stack: + _enter_relay_logging_mocks(stack, {}) + stack.enter_context( + patch("litellm.proxy.proxy_server.proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache())) + ) + response: Final = await pass_through_request( + request=request, + target=target, + custom_headers={}, + user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"), + custom_body=custom_body, + guardrails_config=guardrails_config, + ) + relayed: Final = await _relayed_body(response) + finally: + cleanup() + await fake_client.aclose() + + assert response.status_code == 200 + return MappingProxyType(json.loads(relayed)) + + +_MESSAGES_BODY: Final = MappingProxyType( + {"model": "claude-sonnet-4-5", "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]} +) +_ANTHROPIC_MESSAGES: Final = "https://api.anthropic.com/v1/messages" + + +@pytest.fixture +def no_configured_anthropic_base(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"]) +@pytest.mark.parametrize( + "client_metadata, target, forwarded_metadata", + [ + ({"user_id": "user-1"}, _ANTHROPIC_MESSAGES, {"user_id": "user-1"}), + ({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/", {"user_id": "user-1"}), + ({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}?beta=true", {"user_id": "user-1"}), + ( + {"user_id": "user-1", "tags": ["a"], "trace_id": "t", "generation_name": "g", "foo": "bar"}, + _ANTHROPIC_MESSAGES, + {"user_id": "user-1"}, + ), + ({"user_id": "1234567890"}, _ANTHROPIC_MESSAGES, {"user_id": "1234567890"}), + ({"tags": ["a"], "foo": "bar"}, _ANTHROPIC_MESSAGES, None), + ({"user_id": "a@example.com"}, _ANTHROPIC_MESSAGES, None), + ({"user_id": "x" * (ANTHROPIC_MAX_USER_ID_LENGTH + 1)}, _ANTHROPIC_MESSAGES, None), + ({"user_id": 42}, _ANTHROPIC_MESSAGES, None), + ({"user_id": "line one\nline two"}, _ANTHROPIC_MESSAGES, None), + ({"user_id": "a\x1f@example.com"}, _ANTHROPIC_MESSAGES, None), + ({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/count_tokens", None), + ({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/batches", None), + ({"user_id": "user-1"}, "https://api.anthropic.com/v1/models", None), + ({"user_id": "user-1"}, "https://other-vendor.test/anthropic/v1/messages", None), + ({"user_id": "user-1"}, "https://other-vendor.test/v1/responses", None), + ({"user_id": "user-1"}, "https://other-vendor.test/v1/chat/completions", None), + ], + ids=[ + "messages", + "messages-trailing-slash", + "messages-with-query", + "user-id-beside-litellm-and-unknown-keys", + "digits-user-id", + "no-user-id", + "email-user-id", + "over-long-user-id", + "integer-user-id", + "newline-user-id", + "control-character-before-at-sign", + "count-tokens", + "batches", + "other-anthropic-path", + "other-vendor-messages", + "other-vendor-responses", + "other-vendor-chat-completions", + ], +) +async def test_pass_through_forwards_only_an_acceptable_user_id_to_anthropic_messages( + stream: bool, client_metadata: object, target: str, forwarded_metadata: Mapping[str, object] | None +) -> None: + client_body: Final = MappingProxyType( + {**_MESSAGES_BODY, "metadata": client_metadata, **({"stream": True} if stream else {})} + ) + body_without_metadata: Final = {k: v for k, v in client_body.items() if k != "metadata"} + + relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None) + + assert relayed == ( + body_without_metadata if forwarded_metadata is None else {**body_without_metadata, "metadata": forwarded_metadata} + ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +@pytest.mark.parametrize("base_env", ["ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"]) +@pytest.mark.parametrize( + "target, forwarded", + [ + ("http://gateway.test:8080/anthropic/v1/messages", True), + ("http://gateway.test:9090/v1/messages", False), + ("http://other-gateway.test:8080/v1/messages", False), + ], + ids=["configured-upstream", "other-port", "other-host"], +) +async def test_pass_through_treats_the_configured_anthropic_base_as_anthropic( + monkeypatch: pytest.MonkeyPatch, base_env: str, target: str, forwarded: bool +) -> None: + monkeypatch.setenv(base_env, "http://gateway.test:8080/anthropic") + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}}) + + relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None) + + assert relayed == (client_body if forwarded else _MESSAGES_BODY) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +@pytest.mark.parametrize( + "user_id_length, target, forwarded", + [ + (ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, "http://gateway.test/v1/messages", True), + (ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH + 1, "http://gateway.test/v1/messages", False), + (ANTHROPIC_MAX_USER_ID_LENGTH, _ANTHROPIC_MESSAGES, True), + ], + ids=["at-the-configured-base-limit", "over-the-configured-base-limit", "direct-anthropic-keeps-its-limit"], +) +async def test_pass_through_caps_the_user_id_for_a_configured_anthropic_compatible_base( + monkeypatch: pytest.MonkeyPatch, user_id_length: int, target: str, forwarded: bool +) -> None: + monkeypatch.setenv("ANTHROPIC_API_BASE", "http://gateway.test") + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "u" * user_id_length}}) + + relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None) + + assert relayed == (client_body if forwarded else _MESSAGES_BODY) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +@pytest.mark.parametrize( + "default_on, event_hook, guardrails_config, client_fields, stream_scope, forwarded", + [ + (False, GuardrailEventHooks.pre_call, None, {}, None, True), + (False, GuardrailEventHooks.pre_call, {"unregistered-guard": {}}, {}, None, True), + (True, GuardrailEventHooks.post_call, None, {}, None, True), + (True, GuardrailEventHooks.pre_call, None, {}, None, False), + (False, GuardrailEventHooks.pre_call, {"pii-guard": {}}, {}, None, False), + (False, GuardrailEventHooks.pre_call, None, {"litellm_metadata": {"guardrails": ["pii-guard"]}}, None, False), + (False, GuardrailEventHooks.pre_call, {"pii-guard": {}}, {"litellm_metadata": {"guardrails": []}}, None, False), + (True, GuardrailEventHooks.pre_call, None, {"stream": True}, "streaming", False), + (True, GuardrailEventHooks.pre_call, None, {}, "streaming", True), + (True, GuardrailEventHooks.pre_call, None, {"stream": True}, "non_streaming", True), + (True, GuardrailEventHooks.pre_call, None, {}, "non_streaming", False), + ], + ids=[ + "guardrail-not-requested", + "guardrail-not-registered", + "default-on-post-call-guardrail", + "default-on-pre-call-guardrail", + "guardrail-on-the-route", + "guardrail-in-litellm-metadata", + "route-guardrail-beside-an-empty-client-list", + "streaming-guardrail-on-a-stream", + "streaming-guardrail-off-a-stream", + "non-streaming-guardrail-on-a-stream", + "non-streaming-guardrail-off-a-stream", + ], +) +async def test_pass_through_keeps_the_user_id_from_routes_a_guardrail_scans( + default_on: bool, + event_hook: GuardrailEventHooks, + guardrails_config: dict[str, object] | None, + client_fields: Mapping[str, object], + stream_scope: str | None, + forwarded: bool, +) -> None: + guardrail: Final = CustomGuardrail( + guardrail_name="pii-guard", default_on=default_on, event_hook=event_hook, stream_scope=stream_scope + ) + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}, **client_fields}) + body_without_client_fields: Final = {k: v for k, v in client_body.items() if k not in ("metadata", "litellm_metadata")} + + with patch.object(litellm, "callbacks", [guardrail]): + relayed: Final = await _relay_through_pass_through( + client_body, _ANTHROPIC_MESSAGES, guardrails_config=guardrails_config + ) + + assert relayed == ( + {**body_without_client_fields, "metadata": {"user_id": "user-1"}} if forwarded else body_without_client_fields + ) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +async def test_pass_through_warns_without_the_value_when_it_drops_a_refused_user_id( + caplog: pytest.LogCaptureFixture, +) -> None: + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "person@example.com"}}) + + with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name): + await _relay_through_pass_through(client_body, _ANTHROPIC_MESSAGES, guardrails_config=None) + + assert "Not forwarding metadata.user_id" in caplog.text + assert "person@example.com" not in caplog.text + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +async def test_pass_through_drops_metadata_that_is_not_an_object() -> None: + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": "user-1"}) + + relayed: Final = await _relay_through_pass_through(client_body, _ANTHROPIC_MESSAGES, guardrails_config=None) + + assert relayed == _MESSAGES_BODY + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_configured_anthropic_base") +async def test_pass_through_custom_body_replaces_the_client_user_id() -> None: + client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}}) + + relayed: Final = await _relay_through_pass_through( + client_body, _ANTHROPIC_MESSAGES, guardrails_config=None, custom_body=dict(_MESSAGES_BODY) + ) + + assert relayed == _MESSAGES_BODY + + _PARTIAL_RELAY_WARNING_MARKER = "ended before upstream body was fully relayed" diff --git a/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py b/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py index 5013ecd98a2..26ee9f04d01 100644 --- a/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py +++ b/tests/unit/proxy/utils/proxy_logging/test_pre_call_hook.py @@ -996,3 +996,21 @@ async def test_guardrails_only_and_skip_guardrails_are_mutually_exclusive( guardrails_only=True, skip_guardrails=True, ) + + +@pytest.mark.parametrize( + "request_data, expected", + [ + ({"metadata": {"_guardrail_pipelines": ["p1"]}}, True), + ({"metadata": {"user_id": "u"}, "litellm_metadata": {"_guardrail_pipelines": ["p1"]}}, True), + ({"metadata": {"_guardrail_pipelines": 5}}, True), + ({"metadata": {"user_id": "u"}}, False), + ], + ids=["pipeline-in-metadata", "pipeline-in-litellm-metadata", "non-list-pipeline-value", "no-pipeline"], +) +def test_request_has_pre_call_guardrails_reads_pipelines_from_both_metadata_buckets( + proxy_logging, monkeypatch, request_data, expected +): + monkeypatch.setattr(litellm, "callbacks", []) + + assert proxy_logging.request_has_pre_call_guardrails(request_data) is expected From a7b66dd8b9db2c4e2b6e00e9b62faf44d6ddd223 Mon Sep 17 00:00:00 2001 From: Vineeth Sai Varikuntla Date: Sat, 31 Oct 2026 00:29:10 -0700 Subject: [PATCH 6/6] fix(openai): stop stripping cache_control from the caller's own messages and tools (#35602) * fix(openai): stop stripping cache_control from the caller's messages Copy before removing the field, so the caller's own message objects are left alone. * fix(fireworks_ai): strip cache_control on a copy so a fallback keeps the caller's hints * fix: keep the caller's request intact on more providers and trim extra Claude cache hints under drop_params * fix: drop only the oldest extra cache hints inside one message when trimming to the Claude limit * fix: keep replayed session turns yielding their hints as whole turns while a request's own message keeps its newest hints * fix(cache_control): count only hints that reach Claude and cap hints on OpenAI-compatible Claude routes under drop_params * fix(cache_control): keep the capped Claude messages a list in OpenAI-compatible request bodies * fix(caching): cap Claude hints for string drop_params, a Claude base_model, and Converse tool messages The 4-hint cap now reads drop_params through drops_unsupported_params on chat, Converse, and Anthropic routes, names Claude by base_model on OpenAI-compatible routes, and counts a Converse tool message's hints as the one cache point it sends. * fix(caching): trim Claude cache hints in one wire-order pass and reserve a top-level cache_control on OpenAI-compatible routes Dropping the oldest hint one at a time re-walked the whole request per hint, which took seconds on long tool histories. The cap now drops each message's oldest hints in the order the provider counts them, so a string assistant turn loses its text hint before its tool call hints. OpenAI-compatible Claude routes also keep room for a request-level or extra_body cache_control under drop_params. * test: drop narrating comments and stacked generator clauses from cache_control tests * fix(vertex_ai): put every system message in the Gemini context cache so generateContent never sends it beside cachedContent --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../anthropic_cache_control_hook.py | 210 +++++++++++++++++- .../prompt_templates/common_utils.py | 17 +- litellm/llms/anthropic/chat/transformation.py | 40 +++- .../bedrock/chat/converse_transformation.py | 35 ++- litellm/llms/deepinfra/chat/transformation.py | 5 +- .../llms/fireworks_ai/chat/transformation.py | 38 ++-- litellm/llms/groq/chat/transformation.py | 9 +- .../llms/hosted_vllm/chat/transformation.py | 3 +- litellm/llms/mistral/chat/transformation.py | 3 +- .../llms/openai/chat/gpt_transformation.py | 186 +++++++++------- .../openai/chat/o_series_transformation.py | 9 +- litellm/llms/openai/common_utils.py | 44 +++- .../llms/openai/responses/transformation.py | 53 ++--- .../context_caching/transformation.py | 43 ++-- litellm/llms/watsonx/chat/transformation.py | 3 +- litellm/llms/xai/chat/transformation.py | 4 +- .../transformation.py | 105 ++------- ...ore_utils_prompt_templates_common_utils.py | 14 ++ .../test_anthropic_chat_transformation.py | 184 +++++++++++++++ .../chat/test_converse_transformation.py | 83 +++++++ .../test_databricks_chat_transformation.py | 63 ++++++ .../test_deepinfra_chat_transformation.py | 37 +++ .../test_fireworks_ai_chat_transformation.py | 61 +++++ .../chat/test_groq_chat_transformation.py | 17 ++ .../test_hosted_vllm_chat_transformation.py | 42 ++++ .../test_mistral_chat_transformation.py | 18 +- .../chat/test_openai_gpt_transformation.py | 138 ++++++++++++ .../test_openai_responses_transformation.py | 62 +++++- .../openai/test_o_series_transformation.py | 21 ++ .../test_openrouter_chat_transformation.py | 55 +++++ .../test_vertex_ai_context_caching.py | 44 ++++ tests/unit/llms/watsonx/test_watsonx.py | 42 ++++ .../llms/xai/test_xai_chat_transformation.py | 42 ++++ .../test_litellm_completion_responses.py | 11 + 34 files changed, 1463 insertions(+), 278 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index d816bc661f3..39368dee47c 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -14,7 +14,7 @@ import itertools import os import re from collections.abc import Callable, Container, Iterable, Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, cast +from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast from urllib.parse import urlparse from pydantic import TypeAdapter, ValidationError @@ -102,6 +102,192 @@ def _validated_object_list(value: object) -> list[object] | None: return None +_MessageT: Final = TypeVar("_MessageT") +_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None] +_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls") + + +def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]: + lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS) + return tuple( + itertools.chain.from_iterable( + _OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list) + ) + ) + + +def _block_carries_cache_control(block: object) -> bool: + return isinstance(block, dict) and _CONTENT_PART_ADAPTER.validate_python(block).get("cache_control") is not None + + +_ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT: Final = frozenset({"tool", "function"}) + + +def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool: + if fields.get("cache_control") is None: + return False + return ( + not isinstance(fields.get("content"), list) + or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT + ) + + +def cache_control_count(message: object) -> int: + if not isinstance(message, dict): + return 0 + fields: Final = _CONTENT_PART_ADAPTER.validate_python(message) + block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block)) + return int(_message_mark_reaches_provider(fields)) + block_marks + + +def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]: + others: Final = {key: value for key, value in fields.items() if key != "cache_control"} + if "cache_control" not in fields: + return others + cache_control: Final = rewrite(fields["cache_control"]) + return others if cache_control is None else {**others, "cache_control": cache_control} + + +def block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object: + if not isinstance(block, dict): + return block + return _fields_with_cache_control(_CONTENT_PART_ADAPTER.validate_python(block), rewrite) + + +def with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT: + if not isinstance(message, dict): + return message + fields: Final = _CONTENT_PART_ADAPTER.validate_python(message) + blocks_rewritten: Final = { + key: ( + [block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)] + if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list) + else value + ) + for key, value in fields.items() + } + return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped + _MessageT, _fields_with_cache_control(blocks_rewritten, rewrite) + ) + + +def dropped_cache_control(_: object) -> None: + return None + + +def _first_marked_index(value: object) -> int | None: + if not isinstance(value, list): + return None + blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value) + return next((index for index, block in enumerate(blocks) if _block_carries_cache_control(block)), None) + + +_MarkSlot: TypeAlias = tuple[str, int] +_MESSAGE_MARK_SLOT: Final[_MarkSlot] = ("cache_control", -1) + + +def _block_mark_slots(fields: Mapping[str, object], key: str) -> tuple[_MarkSlot, ...]: + value: Final = fields.get(key) + if not isinstance(value, list): + return () + blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value) + return tuple((key, index) for index, block in enumerate(blocks) if _block_carries_cache_control(block)) + + +def _mark_slots_in_wire_order(fields: Mapping[str, object]) -> tuple[_MarkSlot, ...]: + content_slots: Final = _block_mark_slots(fields, "content") + tool_call_slots: Final = _block_mark_slots(fields, "tool_calls") + if not _message_mark_reaches_provider(fields): + return content_slots + tool_call_slots + if isinstance(fields.get("content"), list): + return content_slots + (_MESSAGE_MARK_SLOT,) + tool_call_slots + return (_MESSAGE_MARK_SLOT,) + content_slots + tool_call_slots + + +def _blocks_without_marks(key: str, value: object, dropped_slots: frozenset[_MarkSlot]) -> object: + if key not in _MARK_CARRYING_MESSAGE_FIELDS or not isinstance(value, list): + return value + return [ + block_with_cache_control(block, dropped_cache_control) if (key, index) in dropped_slots else block + for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(value)) + ] + + +def _without_oldest_marks(message: _MessageT, dropped: int) -> _MessageT: + if dropped == 0 or not isinstance(message, dict): + return message + fields: Final = _CONTENT_PART_ADAPTER.validate_python(message) + dropped_slots: Final = frozenset(_mark_slots_in_wire_order(fields)[:dropped]) + blocks_unmarked: Final = {key: _blocks_without_marks(key, value, dropped_slots) for key, value in fields.items()} + return cast( # cast-ok: the same message with its oldest wire-order marks dropped + _MessageT, + _fields_with_cache_control(blocks_unmarked, dropped_cache_control) + if _MESSAGE_MARK_SLOT in dropped_slots + else blocks_unmarked, + ) + + +def tool_message_with_single_cache_mark(message: _MessageT) -> _MessageT: + if not isinstance(message, dict): + return message + fields: Final = _CONTENT_PART_ADAPTER.validate_python(message) + content: Final = fields.get("content") + if fields.get("role") != "tool" or not isinstance(content, list): + return message + kept_index: Final = None if fields.get("cache_control") is not None else _first_marked_index(content) + return cast( # cast-ok: the same tool message keeping only the mark its single cache point reads + _MessageT, + { + **fields, + "content": [ + block if index == kept_index else block_with_cache_control(block, dropped_cache_control) + for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(content)) + ], + }, + ) + + +_BudgetSpend: TypeAlias = Callable[[tuple[int, int], int], tuple[int, int]] + + +def _newest_marks_kept(state: tuple[int, int], marks: int) -> tuple[int, int]: + remaining, _ = state + kept: Final = min(marks, remaining) + return remaining - kept, marks - kept + + +def _whole_message_kept(state: tuple[int, int], marks: int) -> tuple[int, int]: + remaining, _ = state + if marks > remaining: + return remaining, marks + return remaining - marks, 0 + + +def _messages_within_budget( + messages: Sequence[_MessageT], reserved_marks: int, spend: _BudgetSpend +) -> Sequence[_MessageT]: + budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0) + newest_first: Final = tuple(reversed(messages)) + budget_after_each: Final = itertools.accumulate( + (cache_control_count(message) for message in newest_first), spend, initial=(budget, 0) + ) + dropped_counts: Final = tuple(dropped for _, dropped in itertools.islice(budget_after_each, 1, None)) + kept_newest_first: Final = tuple( + _without_oldest_marks(message, dropped) for message, dropped in zip(newest_first, dropped_counts) + ) + return tuple(reversed(kept_newest_first)) + + +def newest_marks_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]: + return _messages_within_budget(messages, reserved_marks, _newest_marks_kept) + + +def whole_messages_within_cache_control_limit( + messages: Sequence[_MessageT], reserved_marks: int +) -> Sequence[_MessageT]: + return _messages_within_budget(messages, reserved_marks, _whole_message_kept) + + def _part_with_cache_control_renamed(part: object) -> object: if not _accepts_prompt_cache_breakpoint(part): return part @@ -1439,3 +1625,25 @@ class AnthropicCacheControlHook(CustomPromptManagement): llm_router=None, ) return None + + +def messages_within_cache_control_cap( + messages: Sequence[_MessageT], tools: object, cache_control: object = None, request_kwargs: object = None +) -> Sequence[_MessageT]: + """Under drop_params, drop the oldest message hints the provider's cache_control cap has no room for. + + Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first. + """ + reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints( + _validated_object_list(tools), cache_control, request_kwargs + ) + kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks) + dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept)) + if dropped > 0: + verbose_logger.warning( + "Dropped %d cache_control hint(s) from the oldest messages because drop_params is on and the " + "provider accepts at most %d cache_control blocks per request", + dropped, + MAX_CACHE_CONTROL_BLOCKS, + ) + return kept diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 862e61cb631..e733f48ffd7 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -88,17 +88,22 @@ def handle_any_messages_to_chat_completion_str_messages_conversion( return [{"input": str(messages)}] +def _message_with_str_content(message: AllMessageValues) -> AllMessageValues: + texts: Final = convert_content_list_to_str(message=message) + if not texts: + return message + return cast( # cast-ok: the same message with its content flattened to one string + AllMessageValues, {**message, "content": texts} + ) + + def handle_messages_with_content_list_to_str_conversion( messages: list[AllMessageValues], ) -> list[AllMessageValues]: """ - Handles messages with content list conversion + Returns new messages with each content list flattened to a string, leaving the caller's messages untouched """ - for message in messages: - texts = convert_content_list_to_str(message=message) - if texts: - message["content"] = texts - return messages + return [_message_with_str_content(message) for message in messages] def strip_name_from_message(message: AllMessageValues, allowed_name_roles: list[str] = ["user"]) -> AllMessageValues: diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index d773e31c733..25245c347f4 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -167,6 +167,7 @@ _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool) _TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _TOOLS_ADAPTER: Final = TypeAdapter(list[object]) _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) +_REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) _NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset( ("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid") ) @@ -2254,6 +2255,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): headers=headers, ) + @staticmethod + def _messages_within_cache_control_cap( + messages: Sequence[AllMessageValues], optional_params: Mapping[str, object] + ) -> Sequence[AllMessageValues]: + from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap + + return messages_within_cache_control_cap( + messages, optional_params.get("tools"), optional_params.get("cache_control"), optional_params + ) + def transform_request( self, model: str, @@ -2271,9 +2282,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): anthropic_messages_pt, ) - drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True + drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params")) + capped_messages: Final = ( + list( + self._messages_within_cache_control_cap( + messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params) + ) + ) + if drop_params + else messages + ) - if "tools" not in optional_params and messages is not None and has_tool_call_blocks(messages): + if "tools" not in optional_params and has_tool_call_blocks(capped_messages): optional_params["tools"], _ = self._map_tools(add_dummy_tool(custom_llm_provider="anthropic")) # Drop thinking param if thinking is enabled but thinking_blocks are missing @@ -2285,9 +2305,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # Related issue: https://github.com/BerriAI/litellm/issues/18926 if ( optional_params.get("thinking") is not None - and messages is not None - and last_assistant_with_tool_calls_has_no_thinking_blocks(messages) - and not any_assistant_message_has_thinking_blocks(messages) + and last_assistant_with_tool_calls_has_no_thinking_blocks(capped_messages) + and not any_assistant_message_has_thinking_blocks(capped_messages) ): if litellm.modify_params: optional_params.pop("thinking", None) @@ -2332,15 +2351,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): _name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request( optional_params=optional_params, ) - if _name_forward_map: - messages = self._rewrite_tool_names_in_messages(messages, _name_forward_map) + request_messages: Final = ( + self._rewrite_tool_names_in_messages(capped_messages, _name_forward_map) + if _name_forward_map + else capped_messages + ) if _name_reverse_map and isinstance(litellm_params, dict): litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map # Only the leading system run becomes the top-level system prompt. A later # system message stays in the conversation: hoisting it rewrites the cached # prefix and re-bills the whole history at cache-write pricing (#36559). - leading_system_run, later_messages = split_leading_system_run(messages) + leading_system_run, later_messages = split_leading_system_run(request_messages) anthropic_system_message_list: Final = self.translate_system_message(messages=list(leading_system_run)) # Handling anthropic API Prompt Caching if len(anthropic_system_message_list) > 0: @@ -2362,7 +2384,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): except Exception as e: raise AnthropicError( status_code=400, - message=f"{e}\nReceived Messages={messages}", + message=f"{e}\nReceived Messages={request_messages}", ) # don't use verbose_logger.exception, if exception is raised self.update_headers_with_optional_anthropic_beta( diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 3800489ec05..21af97e523e 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -25,6 +25,7 @@ from litellm.constants import ( ) from litellm.litellm_core_utils.core_helpers import ( as_str_mapping, + drops_unsupported_params, filter_exceptions_from_params, filter_internal_params, map_finish_reason, @@ -2129,6 +2130,24 @@ class AmazonConverseConfig(BaseConfig): return data + @staticmethod + def _claude_messages_within_cache_control_cap( + messages: Sequence[AllMessageValues], + capability_model: str, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], + ) -> Sequence[AllMessageValues]: + if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower(): + return messages + from litellm.integrations.anthropic_cache_control_hook import ( + messages_within_cache_control_cap, + tool_message_with_single_cache_mark, + ) + + return messages_within_cache_control_cap( + tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools") + ) + async def _async_transform_request( self, model: str, @@ -2140,7 +2159,13 @@ class AmazonConverseConfig(BaseConfig): base_model: Final = litellm_params.get("base_model") capability_model: Final = bedrock_capability_model(model, base_model) messages, system_content_blocks = self._transform_system_message( - messages, model=capability_model, deployment_model=model + list( + self._claude_messages_within_cache_control_cap( + messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params + ) + ), + model=capability_model, + deployment_model=model, ) # Convert last user message to guarded_text if guardrailConfig is present @@ -2221,7 +2246,13 @@ class AmazonConverseConfig(BaseConfig): base_model: Final = litellm_params.get("base_model") capability_model: Final = bedrock_capability_model(model, base_model) messages, system_content_blocks = self._transform_system_message( - messages, model=capability_model, deployment_model=model + list( + self._claude_messages_within_cache_control_cap( + messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params + ) + ), + model=capability_model, + deployment_model=model, ) # Convert last user message to guarded_text if guardrailConfig is present diff --git a/litellm/llms/deepinfra/chat/transformation.py b/litellm/llms/deepinfra/chat/transformation.py index 004128c0184..7caef58dc79 100644 --- a/litellm/llms/deepinfra/chat/transformation.py +++ b/litellm/llms/deepinfra/chat/transformation.py @@ -130,7 +130,8 @@ class DeepInfraConfig(OpenAIGPTConfig): - Input: {"role": "tool", "content": [{"type": "text", "text": "result"}]} - Output: {"role": "tool", "content": "[{\"type\": \"text\", \"text\": \"result\"}]"} """ - for message in messages: + copied_messages: Final = [message.copy() for message in messages] + for message in copied_messages: if message.get("role") == "tool": content = message.get("content") @@ -149,7 +150,7 @@ class DeepInfraConfig(OpenAIGPTConfig): # For complex content, serialize the entire array as JSON string message["content"] = json.dumps(content) - return messages + return copied_messages @overload def _transform_messages( diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 92c6c8439b3..5a9e5745c40 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -23,6 +23,7 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionToolParam, + ChatCompletionToolParamFunctionChunk, OpenAIChatCompletionToolParam, ) from litellm.types.utils import ( @@ -45,6 +46,7 @@ from ...openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, OpenAIGPTConfig, ) +from ...openai.common_utils import without_cache_control from ..common_utils import ( FIREROUTER, FireworksAIException, @@ -480,15 +482,23 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): return (("response_format", _json_schema_response_format(choice_schema, "choice")),) def _transform_tools(self, tools: list[OpenAIChatCompletionToolParam]) -> list[OpenAIChatCompletionToolParam]: - for tool in tools: - if tool.get("type") != "function": - continue - function = tool["function"] - function.pop("strict", None) - params = function.get("parameters") - if isinstance(params, dict): - unpack_legacy_defs(params) - return tools + return [self._fireworks_tool(tool) for tool in tools] + + @staticmethod + def _fireworks_tool(tool: OpenAIChatCompletionToolParam) -> OpenAIChatCompletionToolParam: + if tool.get("type") != "function": + return tool + function: Final = tool["function"] + params: Final = function.get("parameters") + unstrict_function: Final = cast( # cast-ok: the same function definition minus `strict` + ChatCompletionToolParamFunctionChunk, {key: value for key, value in function.items() if key != "strict"} + ) + new_function: Final[ChatCompletionToolParamFunctionChunk] = ( + {**unstrict_function, "parameters": unpack_legacy_defs(params, copy=True)} + if isinstance(params, dict) + else unstrict_function + ) + return {**tool, "function": new_function} def _transform_messages_helper( self, messages: list[AllMessageValues], model: str, litellm_params: dict @@ -496,10 +506,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): """ Strip fields not permitted by FireworksAI from messages. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - filter_value_from_dict, - ) - supports_vision_value: Final = self._get_model_cost_capability_exact(model=model, capability="supports_vision") for message in messages: if message["role"] == "user": @@ -530,9 +536,11 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): model=model, llm_provider="fireworks_ai", ) - filter_value_from_dict(cast(dict, message), "cache_control") - return [strip_litellm_response_only_fields(message) for message in messages] + return [ + strip_litellm_response_only_fields(cast(AllMessageValues, without_cache_control(message))) + for message in messages + ] # Cached index of fireworks_ai/* entries from litellm.model_cost. Building # this index requires a full scan of model_cost (tens of thousands of diff --git a/litellm/llms/groq/chat/transformation.py b/litellm/llms/groq/chat/transformation.py index 23f68335ea9..fd6b81fce8a 100644 --- a/litellm/llms/groq/chat/transformation.py +++ b/litellm/llms/groq/chat/transformation.py @@ -140,7 +140,8 @@ class GroqChatConfig(OpenAILikeChatConfig): def _transform_messages( self, messages: list[AllMessageValues], model: str, is_async: bool = False ) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]: - for idx, message in enumerate(messages): + transformed_messages: Final = list(messages) + for idx, message in enumerate(transformed_messages): """ 1. Don't pass 'null' function_call assistant message to groq - https://github.com/BerriAI/litellm/issues/5839 """ @@ -154,12 +155,12 @@ class GroqChatConfig(OpenAILikeChatConfig): for k, v in _message.items(): if v is not None: new_message[k] = v - messages[idx] = strip_litellm_internal_message_fields(new_message) + transformed_messages[idx] = strip_litellm_internal_message_fields(new_message) if is_async: - return super()._transform_messages(messages=messages, model=model, is_async=True) + return super()._transform_messages(messages=transformed_messages, model=model, is_async=True) else: - return super()._transform_messages(messages=messages, model=model, is_async=False) + return super()._transform_messages(messages=transformed_messages, model=model, is_async=False) def transform_messages( self, diff --git a/litellm/llms/hosted_vllm/chat/transformation.py b/litellm/llms/hosted_vllm/chat/transformation.py index 9f9503f93b6..3386cd452da 100644 --- a/litellm/llms/hosted_vllm/chat/transformation.py +++ b/litellm/llms/hosted_vllm/chat/transformation.py @@ -2,6 +2,7 @@ Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions` """ +import copy import json from collections.abc import Coroutine from typing import Final, Literal, cast, overload @@ -88,7 +89,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig): ) -> dict: _tools = non_default_params.pop("tools", None) if _tools is not None: - _tools = remove_additional_properties(_tools) + _tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params _tools = remove_strict_from_schema(_tools) if isinstance(_tools, list): _tools = self._convert_custom_tools_to_function_tools(_tools) diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index db8dd3fa7f0..1ecef46ffb1 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -755,7 +755,8 @@ class MistralConfig(OpenAIGPTConfig): if _name is not None: # Remove name if not a tool message if message["role"] != "tool" or isinstance(_name, str) and len(_name.strip()) == 0: - message.pop("name", None) + message_without_name: Final = {k: v for k, v in message.items() if k != "name"} + return cast(AllMessageValues, message_without_name) # cast-ok: the same message minus name return message diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 22af51aaabe..9d551f6f859 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -14,7 +14,7 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS -from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( extract_reasoning_content, handle_invalid_parallel_tool_calls, @@ -64,7 +64,7 @@ from litellm.types.utils import ( ) from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support -from ..common_utils import OpenAIError +from ..common_utils import OpenAIError, without_cache_control from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config if TYPE_CHECKING: @@ -128,6 +128,29 @@ def _litellm_params_str(litellm_params: Mapping[str, object] | None, key: str) - return value if isinstance(value, str) else None +def _drops_params(litellm_params: Mapping[str, object]) -> bool: + return drops_unsupported_params(litellm_params.get("drop_params")) + + +def _names_claude(model: str, litellm_params: Mapping[str, object]) -> bool: + base_model: Final = litellm_params.get("base_model") + return "claude" in model.lower() or (isinstance(base_model, str) and "claude" in base_model.lower()) + + +def _claude_messages_within_cache_control_cap( + model: str, + messages: Sequence[AllMessageValues], + tools: Sequence["ChatCompletionToolParam"] | None, + litellm_params: Mapping[str, object], + request_params: Mapping[str, object], +) -> Sequence[AllMessageValues]: + if not _names_claude(model, litellm_params) or not _drops_params(litellm_params): + return messages + from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap + + return list(messages_within_cache_control_cap(messages, tools, request_params.get("cache_control"), request_params)) + + def _prompt_cache_options(optional_params: Mapping[str, object]) -> object | None: try: extra_body: Final = _OBJECT_MAPPING.validate_python(optional_params.get("extra_body")) @@ -311,52 +334,58 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): return content_copy async def _async_handle_pdf_url(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile: - file_id: Final = content_item.get("file_id") + content_copy: Final = content_item.copy() + file_id: Final = content_copy.get("file_id") if file_id is not None: # check for file id being url done in _handle_pdf_url base64_data: Final = await async_convert_url_to_base64(file_id) - content_item["file_data"] = base64_data - content_item["filename"] = "my_file.pdf" - content_item.pop("file_id") - return content_item + content_copy["file_data"] = base64_data + content_copy["filename"] = "my_file.pdf" + content_copy.pop("file_id") + return content_copy def _common_file_data_check(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile: file_data: Final = content_item.get("file_data") filename: Final = content_item.get("filename") if file_data is not None and filename is None: - content_item["filename"] = "my_file.pdf" + return {**content_item, "filename": "my_file.pdf"} return content_item + @staticmethod + def _image_item_for_openai(content_item: ChatCompletionImageObject) -> ChatCompletionImageObject: + image_url: Final = content_item["image_url"] + if isinstance(image_url, str): + return {**content_item, "image_url": {"url": image_url}} + if not isinstance(image_url, dict): + return content_item + new_image_url_obj: Final = cast( + ChatCompletionImageUrlObject, + {k: v for k, v in image_url.items() if k not in {"format", "video_metadata"}}, + ) + return {**content_item, "image_url": new_image_url_obj} + + @staticmethod + def _file_item_for_openai(content_item: ChatCompletionFileObject) -> ChatCompletionFileObject: + litellm_specific_params: Final = {"format", "detail", "video_metadata"} + file_obj: Final = content_item.get("file") + if file_obj is None: + raise litellm.BadRequestError( + message="Content block has type='file' but is missing the required 'file' field", + model=None, + llm_provider="openai", + ) + new_file_obj: Final = ChatCompletionFileObjectFile( + **{k: v for k, v in file_obj.items() if k not in litellm_specific_params} + ) + return {**content_item, "file": new_file_obj} + def _apply_common_transform_content_item( self, content_item: OpenAIMessageContentListBlock, ) -> OpenAIMessageContentListBlock: - litellm_specific_params: Final = {"format", "detail", "video_metadata"} if content_item.get("type") == "image_url": - content_item = cast(ChatCompletionImageObject, content_item) - if isinstance(content_item["image_url"], str): - content_item["image_url"] = { - "url": content_item["image_url"], - } - elif isinstance(content_item["image_url"], dict): - new_image_url_obj: Final = cast( - ChatCompletionImageUrlObject, - {k: v for k, v in content_item["image_url"].items() if k not in {"format", "video_metadata"}}, - ) - content_item["image_url"] = new_image_url_obj - elif content_item.get("type") == "file": - content_item = cast(ChatCompletionFileObject, content_item) - file_obj: Final = content_item.get("file") - if file_obj is None: - raise litellm.BadRequestError( - message="Content block has type='file' but is missing the required 'file' field", - model=None, - llm_provider="openai", - ) - new_file_obj: Final = ChatCompletionFileObjectFile( - **{k: v for k, v in file_obj.items() if k not in litellm_specific_params} - ) - content_item["file"] = new_file_obj - + return self._image_item_for_openai(cast(ChatCompletionImageObject, content_item)) + if content_item.get("type") == "file": + return self._file_item_for_openai(cast(ChatCompletionFileObject, content_item)) return content_item def _transform_content_item( @@ -419,31 +448,34 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): hoisted_messages: Final = hoist_images_from_tool_messages(stripped_messages) async def _async_transform(): - for message in hoisted_messages: - message_content = message.get("content") - message_role = message.get("role") - - if message_role == "user" and message_content and isinstance(message_content, list): - message_content_types = cast(list[OpenAIMessageContentListBlock], message_content) - for i, content_item in enumerate(message_content_types): - message_content_types[i] = await self._async_transform_content_item( - cast(OpenAIMessageContentListBlock, content_item), - ) - return hoisted_messages + return [await self._async_user_message_for_openai(message) for message in hoisted_messages] if is_async: return _async_transform() - else: - for message in hoisted_messages: - message_content = message.get("content") - message_role = message.get("role") - if message_role == "user" and message_content and isinstance(message_content, list): - message_content_types = cast(list[OpenAIMessageContentListBlock], message_content) - for i, content_item in enumerate(message_content): - message_content_types[i] = self._transform_content_item( - cast(OpenAIMessageContentListBlock, content_item) - ) - return hoisted_messages + return [self._user_message_for_openai(message) for message in hoisted_messages] + + @staticmethod + def _user_content_parts(message: AllMessageValues) -> Sequence[OpenAIMessageContentListBlock] | None: + message_content: Final = message.get("content") + if message.get("role") != "user" or not message_content or not isinstance(message_content, list): + return None + return cast(Sequence[OpenAIMessageContentListBlock], message_content) + + def _user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues: + parts: Final = self._user_content_parts(message) + if parts is None: + return message + return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI + AllMessageValues, {**message, "content": [self._transform_content_item(part) for part in parts]} + ) + + async def _async_user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues: + parts: Final = self._user_content_parts(message) + if parts is None: + return message + return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI + AllMessageValues, {**message, "content": [await self._async_transform_content_item(part) for part in parts]} + ) def transform_messages( self, @@ -462,23 +494,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): messages: list[AllMessageValues], tools: list["ChatCompletionToolParam"] | None = None, ) -> tuple[list[AllMessageValues], list["ChatCompletionToolParam"] | None]: - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - filter_value_from_dict, - ) from litellm.types.llms.openai import ChatCompletionToolParam - for i, message in enumerate(messages): - messages[i] = cast( - AllMessageValues, - filter_value_from_dict(message, "cache_control"), - ) - if tools is not None: - for i, tool in enumerate(tools): - tools[i] = cast( - ChatCompletionToolParam, - filter_value_from_dict(tool, "cache_control"), - ) - return messages, tools + new_messages: Final = [cast(AllMessageValues, without_cache_control(message)) for message in messages] + new_tools: Final = ( + [cast(ChatCompletionToolParam, without_cache_control(tool)) for tool in tools] + if tools is not None + else None + ) + return new_messages, new_tools def translate_developer_role_to_system_role( self, @@ -527,6 +551,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): tools: Sequence["ChatCompletionToolParam"] | None, prompt_cache_options: object, litellm_params: Mapping[str, object], + request_params: Mapping[str, object], ) -> tuple[Sequence[AllMessageValues], Sequence["ChatCompletionToolParam"] | None]: """ A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on @@ -543,10 +568,11 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): model, custom_llm_provider, api_base, prompt_cache_options ) if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base): - return messages, tools - drops_breakpoints: Final = not reads_breakpoints and ( - litellm.drop_params is True or litellm_params.get("drop_params") is True - ) + return ( + _claude_messages_within_cache_control_cap(model, messages, tools, litellm_params, request_params), + tools, + ) + drops_breakpoints: Final = not reads_breakpoints and _drops_params(litellm_params) renamed: Final = ( [AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages] if reads_breakpoints @@ -561,7 +587,13 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): if drops_breakpoints else list(tools) ) - return self.remove_cache_control_flag_from_messages_and_tools(model=model, messages=renamed, tools=kept_tools) + sent_messages, sent_tools = self.remove_cache_control_flag_from_messages_and_tools( + model=model, messages=renamed, tools=kept_tools + ) + return ( + _claude_messages_within_cache_control_cap(model, sent_messages, sent_tools, litellm_params, request_params), + sent_tools, + ) def _sanitized_tools_update_for_openai( self, @@ -623,6 +655,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): tools=optional_params.get("tools", []), prompt_cache_options=_prompt_cache_options(optional_params), litellm_params=litellm_params, + request_params=optional_params, ) if tools is not None and len(tools) > 0: optional_params["tools"] = tools @@ -654,6 +687,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): tools=optional_params.get("tools", []), prompt_cache_options=_prompt_cache_options(optional_params), litellm_params=litellm_params, + request_params=optional_params, ) if tools is not None and len(tools) > 0: optional_params["tools"] = tools diff --git a/litellm/llms/openai/chat/o_series_transformation.py b/litellm/llms/openai/chat/o_series_transformation.py index 64285aafdb0..4d56a3b38c5 100644 --- a/litellm/llms/openai/chat/o_series_transformation.py +++ b/litellm/llms/openai/chat/o_series_transformation.py @@ -153,12 +153,13 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig): - role: system ==> translate to role 'user' """ _supports_system_messages: Final = supports_system_messages(model, "openai") - for i, message in enumerate(messages): + transformed_messages: Final = list(messages) + for i, message in enumerate(transformed_messages): if message["role"] == "system" and not _supports_system_messages: new_message = ChatCompletionUserMessage(content=message["content"], role="user") - messages[i] = new_message # Replace the old message with the new one + transformed_messages[i] = new_message # Replace the old message with the new one if is_async: - return super()._transform_messages(messages, model, is_async=cast(Literal[True], True)) + return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[True], True)) else: - return super()._transform_messages(messages, model, is_async=cast(Literal[False], False)) + return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[False], False)) diff --git a/litellm/llms/openai/common_utils.py b/litellm/llms/openai/common_utils.py index c113cb351d8..d476cf62626 100644 --- a/litellm/llms/openai/common_utils.py +++ b/litellm/llms/openai/common_utils.py @@ -3,6 +3,7 @@ Common helpers / utils across al OpenAI endpoints """ import asyncio +import copy import hashlib import inspect import json @@ -31,7 +32,7 @@ if TYPE_CHECKING: import litellm from litellm.exceptions import error_response_without_credentials from litellm.litellm_core_utils.completion_timeout import CompletionTimeout -from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object +from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, iter_items from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -43,6 +44,47 @@ from litellm.llms.custom_httpx.http_handler import ( ) +def _is_plain_dict(value: object) -> bool: + return isinstance(value, dict) + + +def _is_list(value: object) -> bool: + return isinstance(value, list) + + +def _contains_cache_control(value: object) -> bool: + pending: Final[list[object]] = [value] # mutable-ok: a local worklist, never returned or stored + while pending: + node = pending.pop() + mapping = as_str_mapping(node) + if mapping is not None: + if "cache_control" in mapping: + return True + pending.extend(mapping.values()) + elif _is_list(node): + pending.extend(iter_items(node)) + return False + + +def without_cache_control(item: object) -> object: + """Return `item` with every `cache_control` key removed, without touching the original. + + `filter_value_from_dict` deletes the key in place and recurses into nested dicts and + lists, so the caller's own message, tool or input object has to be copied first: it + may reuse the same list on a provider that does support prompt caching. The copy is + skipped when there is nothing to strip, which keeps the object identity the + pass-through paths rely on and avoids a deep copy on every request. + """ + from litellm.litellm_core_utils.prompt_templates.common_utils import filter_value_from_dict + + if not _is_plain_dict(item) or not _contains_cache_control(item): + return item + copied: Final = copy.deepcopy(item) + if isinstance(copied, dict): + filter_value_from_dict(copied, "cache_control") + return copied + + def _get_client_init_params(cls: type) -> tuple[str, ...]: """Extract __init__ parameter names (excluding 'self') from a class.""" return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self") diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 9b8515a9ecd..7cca31c5d3f 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -52,7 +52,7 @@ from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS -from ..common_utils import OpenAIError +from ..common_utils import OpenAIError, without_cache_control from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16 @@ -658,13 +658,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search), tools=mapped_tools, ) - if marker_safe_input is not validated_input: - self._strip_caller_input_as_if_unmarked( - model=model, - caller_input=validated_input, - reads_breakpoints=reads_breakpoints, - maps_anthropic_tool_search=maps_anthropic_tool_search, - ) object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools) sanitized_tools: Final = self._sanitized_tool_schemas_for_openai( model=model, tools=object_schema_tools, litellm_params=litellm_params @@ -679,17 +672,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): return marked_input return input_without_unanswered_anthropic_tool_search_calls(marked_input) - def _strip_caller_input_as_if_unmarked( - self, - model: str, - caller_input: str | ResponseInputParam, - reads_breakpoints: bool, - maps_anthropic_tool_search: bool, - ) -> None: - self.remove_cache_control_flag_from_input_and_tools( - model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search) - ) - def _input_with_wire_safe_prompt_cache_breakpoints( self, model: str, @@ -800,24 +782,29 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): the chat path. Strips Anthropic-only `cache_control` markers from Responses API input content blocks and tools. - `filter_value_from_dict` mutates each dict in place, so the same - objects are returned. + `filter_value_from_dict` deletes the key in place and recurses, so each + item is copied first: the caller keeps its own input list and may reuse + it on a provider that does support prompt caching. """ - from litellm.litellm_core_utils.prompt_templates.common_utils import ( - filter_value_from_dict, + new_input: Final = ( + cast( # cast-ok: the comprehension rebuilds the input item for item + ResponseInputParam, + [without_cache_control(item) for item in input], + ) + if isinstance(input, list) + else input ) - if isinstance(input, list): - for item in input: - if isinstance(item, dict): - filter_value_from_dict(cast(dict, item), "cache_control") + new_tools: Final = ( + cast( # cast-ok: the comprehension rebuilds the tool list item for item + Sequence[ALL_RESPONSES_API_TOOL_PARAMS], + [without_cache_control(tool) for tool in tools], + ) + if tools is not None + else None + ) - if tools is not None: - for tool in tools: - if isinstance(tool, dict): - filter_value_from_dict(cast(dict, tool), "cache_control") - - return input, tools + return new_input, new_tools def _drop_bridge_minted_reasoning_items(self, input: str | ResponseInputParam) -> str | ResponseInputParam: if self.custom_llm_provider not in _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING or not isinstance(input, list): diff --git a/litellm/llms/vertex_ai/context_caching/transformation.py b/litellm/llms/vertex_ai/context_caching/transformation.py index f683ad0e377..a4601af944b 100644 --- a/litellm/llms/vertex_ai/context_caching/transformation.py +++ b/litellm/llms/vertex_ai/context_caching/transformation.py @@ -110,38 +110,23 @@ def separate_cached_messages( messages: list[AllMessageValues], ) -> tuple[list[AllMessageValues], list[AllMessageValues]]: """ - Returns separated cached and non-cached messages. - - Args: - messages: List of messages to be separated. - - Returns: - Tuple containing: - - cached_messages: List of cached messages. - - non_cached_messages: List of non-cached messages. + Returns the first continuous block of cached messages, plus every system message once + any message is cached, since generateContent rejects a system_instruction beside a + cachedContent, and the remaining messages. """ - cached_messages: list[AllMessageValues] = [] - non_cached_messages: list[AllMessageValues] = [] + filtered_messages: Final = [(idx, message) for idx, message in enumerate(messages) if is_cached_message(message)] + if not filtered_messages: + return [], messages - # Extract cached messages and their indices - filtered_messages: Final[list[tuple[int, AllMessageValues]]] = [] - for idx, message in enumerate(messages): - if is_cached_message(message=message): - filtered_messages.append((idx, message)) - - # Validate only one block of continuous cached messages last_continuous_block_idx: Final = get_first_continuous_block_idx(filtered_messages) - # Separate messages based on the block of cached messages - if filtered_messages and last_continuous_block_idx is not None: - first_cached_idx: Final = filtered_messages[0][0] - last_cached_idx: Final = filtered_messages[last_continuous_block_idx][0] - - cached_messages = messages[first_cached_idx : last_cached_idx + 1] - non_cached_messages = messages[:first_cached_idx] + messages[last_cached_idx + 1 :] - else: - non_cached_messages = messages - - return cached_messages, non_cached_messages + cached_block: Final = range(filtered_messages[0][0], filtered_messages[last_continuous_block_idx][0] + 1) + goes_in_cache: Final = tuple( + idx in cached_block or message.get("role") == "system" for idx, message in enumerate(messages) + ) + return ( + [message for message, cached in zip(messages, goes_in_cache) if cached], + [message for message, cached in zip(messages, goes_in_cache) if not cached], + ) def cached_messages_end_on_supported_turn(cached_messages: Sequence[AllMessageValues]) -> bool: diff --git a/litellm/llms/watsonx/chat/transformation.py b/litellm/llms/watsonx/chat/transformation.py index 071ff7108b9..25981b0f04c 100644 --- a/litellm/llms/watsonx/chat/transformation.py +++ b/litellm/llms/watsonx/chat/transformation.py @@ -4,6 +4,7 @@ Translation from OpenAI's `/chat/completions` endpoint to IBM WatsonX's `/text/c Docs: https://cloud.ibm.com/apidocs/watsonx-ai#text-chat """ +import copy from typing import Final from litellm import verbose_logger @@ -57,7 +58,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig): _tools = non_default_params.pop("tools", None) if _tools is not None: # remove 'additionalProperties' from tools - _tools = remove_additional_properties(_tools) + _tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params # remove 'strict' from tools _tools = remove_strict_from_schema(_tools) if _tools is not None: diff --git a/litellm/llms/xai/chat/transformation.py b/litellm/llms/xai/chat/transformation.py index 2b943fa4107..f53b71a5a9a 100644 --- a/litellm/llms/xai/chat/transformation.py +++ b/litellm/llms/xai/chat/transformation.py @@ -1,3 +1,4 @@ +import copy from collections.abc import AsyncIterator, Iterator, Mapping from types import MappingProxyType from typing import Any, Final @@ -208,7 +209,8 @@ class XAIChatConfig(OpenAIGPTConfig): tools = [ filtered_tool for filtered_tool in ( - _FILTERED_TOOL.validate_python(filter_value_from_dict(tool, "strict")) for tool in value + _FILTERED_TOOL.validate_python(filter_value_from_dict(copy.deepcopy(tool), "strict")) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params + for tool in value ) if filtered_tool is not None ] diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 8f0a85d67de..2e156d5abba 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -8,7 +8,7 @@ import uuid from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from functools import partial -from itertools import accumulate, chain, groupby, islice +from itertools import chain, groupby from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -45,7 +45,15 @@ from litellm.constants import ( REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER, ) -from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook +from litellm.integrations.anthropic_cache_control_hook import ( + AnthropicCacheControlHook, + block_with_cache_control, + cache_control_count, + dropped_cache_control, + newest_marks_within_cache_control_limit, + whole_messages_within_cache_control_limit, + with_cache_control, +) from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, @@ -603,10 +611,6 @@ def _validated_cache_control(value: object) -> ChatCompletionCachedContent | Non return None -def _dropped_cache_control(_: object) -> None: - return None - - def _without_cache_control_key(item: object) -> object: if not isinstance(item, dict): return item @@ -694,83 +698,6 @@ def _content_blocks_carrying(content: object, cache_control: ChatCompletionCache return _with_cache_control_on_last_block(_OBJECT_LIST_ADAPTER.validate_python(content), cache_control) -_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls") - - -def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]: - lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS) - return tuple( - chain.from_iterable(_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list)) - ) - - -def _block_carries_cache_control(block: object) -> bool: - return isinstance(block, dict) and _STR_KEY_DICT_ADAPTER.validate_python(block).get("cache_control") is not None - - -def _cache_control_count(message: object) -> int: - if not isinstance(message, dict): - return 0 - fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message) - block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block)) - return int(fields.get("cache_control") is not None) + block_marks - - -_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None] - - -def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]: - others: Final = {key: value for key, value in fields.items() if key != "cache_control"} - if "cache_control" not in fields: - return others - cache_control: Final = rewrite(fields["cache_control"]) - return others if cache_control is None else {**others, "cache_control": cache_control} - - -def _block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object: - if not isinstance(block, dict): - return block - return _fields_with_cache_control(_STR_KEY_DICT_ADAPTER.validate_python(block), rewrite) - - -def _with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT: - if not isinstance(message, dict): - return message - fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message) - blocks_rewritten: Final = { - key: ( - [_block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)] - if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list) - else value - ) - for key, value in fields.items() - } - return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped - _MessageT, _fields_with_cache_control(blocks_rewritten, rewrite) - ) - - -def _kept_within_budget(state: tuple[int, bool], marks: int) -> tuple[int, bool]: - remaining, _ = state - if marks > remaining: - return remaining, False - return remaining - marks, True - - -def _messages_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]: - budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0) - newest_first: Final = tuple(reversed(messages)) - budget_after_each: Final = accumulate( - (_cache_control_count(message) for message in newest_first), _kept_within_budget, initial=(budget, True) - ) - kept_flags: Final = tuple(kept for _, kept in islice(budget_after_each, 1, None)) - kept_newest_first: Final = tuple( - message if kept else _with_cache_control(message, _dropped_cache_control) - for message, kept in zip(newest_first, kept_flags) - ) - return tuple(reversed(kept_newest_first)) - - _SYSTEM_MESSAGE_FIELDS: Final = frozenset(("content", "cache_control")) @@ -799,7 +726,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT: if fields is None: return message if fields.get("role") == "tool": - return _with_cache_control(message, _dropped_cache_control) + return with_cache_control(message, dropped_cache_control) tool_calls: Final = fields.get("tool_calls") if not isinstance(tool_calls, list): return message @@ -808,7 +735,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT: { **fields, "tool_calls": [ - _block_with_cache_control(tool_call, _dropped_cache_control) + block_with_cache_control(tool_call, dropped_cache_control) for tool_call in _OBJECT_LIST_ADAPTER.validate_python(tool_calls) ], }, @@ -822,7 +749,7 @@ class _ItemMarkRules: external_marks: int def sanitized(self, message: _MessageT) -> _MessageT: - valid: Final = _with_cache_control(message, _validated_cache_control) + valid: Final = with_cache_control(message, _validated_cache_control) return _without_tool_block_marks(valid) if self.refuses_tool_block_marks else valid def checked(self, message: _MessageT) -> _MessageT: @@ -845,15 +772,15 @@ def _request_messages_for_cache_points(messages: Sequence[_MessageT], rules: _It as_system: Final = tuple(_developer_message_as_system_message(message) for message in messages) if not rules.drop_params: return as_system - return _messages_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks) + return newest_marks_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks) def _replayed_messages_for_cache_points( replayed_messages: Sequence[_MessageT], current_messages: Sequence[object], rules: _ItemMarkRules ) -> Sequence[_MessageT]: as_system: Final = tuple(_developer_message_as_system_message(message) for message in replayed_messages) - current_marks: Final = sum(_cache_control_count(message) for message in current_messages) - return _messages_within_cache_control_limit( + current_marks: Final = sum(cache_control_count(message) for message in current_messages) + return whole_messages_within_cache_control_limit( tuple(map(rules.sanitized, as_system)), current_marks + rules.external_marks ) diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 72e72714cd4..4ce6aee1619 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_semantic_cache_prompt_from_messages, get_str_from_messages, handle_any_messages_to_chat_completion_str_messages_conversion, + handle_messages_with_content_list_to_str_conversion, hoist_images_from_tool_messages, is_encrypted_reasoning_block, merge_consecutive_system_messages, @@ -2675,3 +2676,16 @@ def test_tpm_projection_preserves_non_media_identity_and_opaque_values() -> None projected, media = split_media_for_tpm(request) assert projected is request assert media == () + + +def test_content_list_flattening_returns_new_messages_and_leaves_the_callers_untouched() -> None: + messages: Final = [ + {"role": "user", "content": [{"type": "text", "text": "Hello"}, {"type": "text", "text": " world"}]}, + {"role": "assistant", "content": "Hi"}, + ] + messages_before: Final = copy.deepcopy(messages) + + flattened: Final = handle_messages_with_content_list_to_str_conversion(messages) + + assert flattened == [{"role": "user", "content": "Hello world"}, {"role": "assistant", "content": "Hi"}] + assert messages == messages_before diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index cbb60caa57b..501dd5bd0c6 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -1,6 +1,7 @@ import copy import json import logging +from collections.abc import Iterator from itertools import chain from typing import Final from unittest.mock import MagicMock, patch @@ -9849,3 +9850,186 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key(): assert "cache_control_injection_points" not in data assert "tools" not in data assert litellm_params["metadata"] == {} + + +def _five_cache_hint_conversation() -> list[dict[str, object]]: + return [ + {"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]}, + {"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]}, + {"role": "assistant", "content": "answer 2"}, + {"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]}, + ] + + +def _cached_texts(node: object) -> Iterator[str]: + if isinstance(node, dict): + yield from ((node["text"],) if "cache_control" in node and "text" in node else ()) + yield from chain.from_iterable(map(_cached_texts, node.values())) + elif isinstance(node, list): + yield from chain.from_iterable(map(_cached_texts, node)) + + +@pytest.mark.parametrize( + ("config", "model"), + [ + (AnthropicConfig(), "claude-haiku-5-5"), + (VertexAIAnthropicConfig(), "claude-haiku-5-5"), + (AmazonAnthropicClaudeConfig(), "anthropic.claude-haiku-5-5"), + (AzureAnthropicConfig(), "claude-haiku-5-5"), + ], + ids=["anthropic", "vertex_ai", "bedrock_invoke", "azure_ai"], +) +@pytest.mark.parametrize( + ("drop_params", "expected_cached_texts"), + [ + (True, ["answer 1", "turn 1", "turn 2", "turn 3"]), + ("true", ["answer 1", "turn 1", "turn 2", "turn 3"]), + (False, ["answer 1", "system prompt", "turn 1", "turn 2", "turn 3"]), + ], + ids=[ + "drop_params_keeps_the_newest_four", + "string_drop_params_keeps_the_newest_four", + "without_drop_params_all_five_pass_through", + ], +) +def test_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params( + config: AnthropicConfig, + model: str, + drop_params: bool | str, + expected_cached_texts: list[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + messages: Final = _five_cache_hint_conversation() + messages_before: Final = copy.deepcopy(messages) + + body: Final = config.transform_request( + model=model, + messages=messages, + optional_params={}, + litellm_params={"drop_params": drop_params}, + headers={}, + ) + + assert sorted(_cached_texts(body)) == expected_cached_texts + assert json.dumps(body).count('"cache_control"') == len(expected_cached_texts) + assert messages == messages_before + + +def test_claude_cache_control_cap_keeps_a_cached_tool_and_drops_the_oldest_message_hint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + messages: Final = _five_cache_hint_conversation()[1:] + tool: Final = { + "name": "get_weather", + "description": "Get the weather", + "input_schema": {"type": "object", "properties": {}}, + "cache_control": {"type": "ephemeral"}, + } + + body: Final = AnthropicConfig().transform_request( + model="claude-haiku-5-5", + messages=messages, + optional_params={"tools": [tool]}, + litellm_params={}, + headers={}, + ) + + assert body["tools"][0]["cache_control"] == {"type": "ephemeral"} + assert sorted(_cached_texts(body["messages"])) == ["answer 1", "turn 2", "turn 3"] + + +def test_claude_cache_control_cap_drops_a_string_assistant_turns_text_hint_before_its_tool_call_hints( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + hint: Final = {"type": "ephemeral"} + tool_calls: Final = [ + {"id": f"call_{name}", "type": "function", "function": {"name": name, "arguments": "{}"}, "cache_control": hint} + for name in ("lookup", "search") + ] + follow_up: Final = [ + {"type": "text", "text": "follow-up", "cache_control": hint}, + {"type": "text", "text": "and more", "cache_control": hint}, + ] + messages: Final = [ + {"role": "user", "content": "question"}, + {"role": "assistant", "content": "checking both", "cache_control": hint, "tool_calls": tool_calls}, + {"role": "tool", "tool_call_id": "call_lookup", "content": "found"}, + {"role": "tool", "tool_call_id": "call_search", "content": "none"}, + {"role": "user", "content": follow_up}, + ] + + body: Final = AnthropicConfig().transform_request( + model="claude-haiku-5-5", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + assistant_blocks: Final = body["messages"][1]["content"] + assert [block["type"] for block in assistant_blocks] == ["text", "tool_use", "tool_use"] + assert ["cache_control" in block for block in assistant_blocks] == [False, True, True] + assert json.dumps(body).count('"cache_control"') == 4 + assert json.dumps(body).count('"cache_control"') == 4 + + +def test_claude_cache_control_cap_keeps_the_newest_hints_inside_one_message( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", True) + messages: Final = [ + { + "role": "user", + "content": [ + *({"type": "text", "text": f"part {n}", "cache_control": {"type": "ephemeral"}} for n in range(1, 6)), + {"type": "text", "text": "Reply with the single word OK."}, + ], + } + ] + messages_before: Final = copy.deepcopy(messages) + + body: Final = AnthropicConfig().transform_request( + model="claude-haiku-5-5", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert sorted(_cached_texts(body["messages"])) == ["part 2", "part 3", "part 4", "part 5"] + assert json.dumps(body).count('"cache_control"') == 4 + assert messages == messages_before + + +def test_claude_cache_control_cap_does_not_count_a_message_level_hint_that_list_content_never_sends( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + messages: Final = [ + {"role": "system", "content": "system prompt", "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "turn 1", "cache_control": {"type": "ephemeral"}}, + {"role": "assistant", "content": "answer 1"}, + {"role": "user", "content": "turn 2", "cache_control": {"type": "ephemeral"}}, + {"role": "assistant", "content": "answer 2"}, + { + "role": "user", + "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}], + "cache_control": {"type": "ephemeral"}, + }, + ] + + body: Final = AnthropicConfig().transform_request( + model="claude-haiku-5-5", + messages=messages, + optional_params={}, + litellm_params={"drop_params": True}, + headers={}, + ) + + assert sorted(_cached_texts(body)) == ["system prompt", "turn 1", "turn 2", "turn 3"] + assert json.dumps(body).count('"cache_control"') == 4 diff --git a/tests/unit/llms/bedrock/chat/test_converse_transformation.py b/tests/unit/llms/bedrock/chat/test_converse_transformation.py index 524caaba7a1..9ab7060d766 100644 --- a/tests/unit/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/unit/llms/bedrock/chat/test_converse_transformation.py @@ -11259,3 +11259,86 @@ def test_converse_sends_only_the_header_betas_when_header_and_body_both_carry_be "thinking-binding-controls-2026-08-01", "thinking-display-updates-2026-08-18", ] + + +def _five_cache_hint_conversation() -> list[dict[str, object]]: + return [ + {"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]}, + {"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]}, + {"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]}, + {"role": "assistant", "content": "answer 2"}, + {"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]}, + ] + + +@pytest.mark.parametrize( + ("drop_params", "expected_system_cache_points"), + [(True, 0), ("true", 0), (False, 1)], + ids=[ + "drop_params_drops_the_oldest_hint", + "string_drop_params_drops_the_oldest_hint", + "without_drop_params_all_five_pass_through", + ], +) +def test_converse_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params( + drop_params: bool | str, expected_system_cache_points: int, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + messages: Final = _five_cache_hint_conversation() + messages_before: Final = copy.deepcopy(messages) + + request: Final = AmazonConverseConfig().transform_request( + model="anthropic.claude-haiku-5-5", + messages=messages, + optional_params={}, + litellm_params={"drop_params": drop_params}, + headers={}, + ) + + assert len(list(_cache_points(request["system"]))) == expected_system_cache_points + assert [len(list(_cache_points(message))) for message in request["messages"]] == [1, 1, 1, 0, 1] + assert messages == messages_before + + +def test_converse_claude_tool_message_with_several_marked_blocks_counts_as_one_cache_point_against_the_cap( + monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + messages: Final = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "policy", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "question", "cache_control": {"type": "ephemeral"}}, + ], + }, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [ + {"type": "text", "text": "part 1", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "part 2", "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "part 3", "cache_control": {"type": "ephemeral"}}, + ], + }, + ] + messages_before: Final = copy.deepcopy(messages) + + request: Final = AmazonConverseConfig().transform_request( + model="anthropic.claude-haiku-5-5", + messages=messages, + optional_params={ + "tools": [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}] + }, + litellm_params={"drop_params": True}, + headers={}, + ) + + assert len(list(_cache_points(request["messages"]))) == 3 + assert messages == messages_before diff --git a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py index 19234767b19..7a6f9c7dc18 100644 --- a/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py +++ b/tests/unit/llms/databricks/chat/test_databricks_chat_transformation.py @@ -2773,3 +2773,66 @@ def test_claude_json_schema_stream_split_across_json_tools_keeps_the_non_stream_ choices: Final = _json_schema_stream_choices(respx_mock, call, split_stream) assert [choice.finish_reason for choice in choices if choice.finish_reason] == [answer.finish_reason] + + +@pytest.mark.parametrize( + ("model", "base_model"), + [("databricks-claude-haiku-5-5", None), ("support-bot-endpoint", "databricks/databricks-claude-haiku-5-5")], + ids=["claude_in_the_endpoint_name", "claude_named_only_by_base_model"], +) +def test_databricks_claude_request_over_the_cache_control_cap_keeps_the_newest_four_hints_under_drop_params( + model: str, base_model: str | None, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + hint: Final = {"type": "ephemeral"} + messages: Final = [ + {"role": "system", "content": "system prompt", "cache_control": hint}, + {"role": "user", "content": "turn 1", "cache_control": hint}, + {"role": "assistant", "content": "answer 1", "cache_control": hint}, + {"role": "user", "content": "turn 2", "cache_control": hint}, + {"role": "assistant", "content": "answer 2"}, + {"role": "user", "content": "turn 3", "cache_control": hint}, + ] + messages_before: Final = copy.deepcopy(messages) + + body: Final = DatabricksConfig().transform_request( + model=model, + messages=messages, + optional_params={}, + litellm_params={"drop_params": True, "custom_llm_provider": "databricks", "base_model": base_model}, + headers={}, + ) + + assert "cache_control" not in json.dumps(body["messages"][0]) + assert json.dumps(body["messages"]).count('"cache_control"') == 4 + assert messages == messages_before + + +@pytest.mark.parametrize( + "optional_params", + [{"cache_control": {"type": "ephemeral"}}, {"extra_body": {"cache_control": {"type": "ephemeral"}}}], + ids=["top_level_cache_control", "extra_body_cache_control"], +) +def test_databricks_claude_cache_control_cap_keeps_room_for_a_top_level_cache_control_under_drop_params( + optional_params: dict[str, object], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + hint: Final = {"type": "ephemeral"} + messages: Final = [ + {"role": "user", "content": "turn 1", "cache_control": hint}, + {"role": "assistant", "content": "answer 1", "cache_control": hint}, + {"role": "user", "content": "turn 2", "cache_control": hint}, + {"role": "assistant", "content": "answer 2", "cache_control": hint}, + {"role": "user", "content": "turn 3"}, + ] + + body: Final = DatabricksConfig().transform_request( + model="databricks-claude-haiku-5-5", + messages=messages, + optional_params=copy.deepcopy(optional_params), + litellm_params={"drop_params": True, "custom_llm_provider": "databricks"}, + headers={}, + ) + + assert "cache_control" not in json.dumps(body["messages"][0]) + assert json.dumps(body["messages"]).count('"cache_control"') == 3 diff --git a/tests/unit/llms/deepinfra/test_deepinfra_chat_transformation.py b/tests/unit/llms/deepinfra/test_deepinfra_chat_transformation.py index 7ec9216df7b..773c1d056f0 100644 --- a/tests/unit/llms/deepinfra/test_deepinfra_chat_transformation.py +++ b/tests/unit/llms/deepinfra/test_deepinfra_chat_transformation.py @@ -1,6 +1,8 @@ import asyncio +import copy import json import os +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -200,3 +202,38 @@ async def test_deepinfra_tool_message_content_transformation_async(): print(f"āœ“ Async test passed: {tool_message['content']}") print("\nāœ… DeepInfra async tool message transformation test passed!") + + +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +async def test_deepinfra_request_stringifies_tool_content_without_touching_the_callers_message(is_async: bool) -> None: + from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig + + config: Final = DeepInfraConfig() + messages: Final = [ + {"role": "user", "content": "Calculate 10 + 10"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "calculator", "arguments": "{}"}} + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": [{"type": "text", "text": "20"}]}, + ] + messages_before: Final = copy.deepcopy(messages) + request_kwargs: Final = { + "model": "deepseek-ai/DeepSeek-V3.2", + "messages": messages, + "optional_params": {}, + "litellm_params": {}, + "headers": {}, + } + + body: Final = ( + await config.async_transform_request(**request_kwargs) + if is_async + else config.transform_request(**request_kwargs) + ) + + assert body["messages"][2]["content"] == "20" + assert messages == messages_before diff --git a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index ed166d206c8..dd352aac66d 100644 --- a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -1,3 +1,4 @@ +import copy import json import logging from collections.abc import Callable @@ -537,6 +538,26 @@ def test_tool_search_reply_sent_to_fireworks_drops_litellm_response_fields(): } +def test_cache_control_is_stripped_from_the_request_but_kept_on_the_callers_messages(): + hint: Final = {"type": "ephemeral"} + messages: Final = [ + {"role": "system", "content": [{"type": "text", "text": "a long cached prompt", "cache_control": hint}]}, + {"role": "user", "content": "hi", "cache_control": hint}, + ] + caller_messages_before: Final = json.dumps(messages) + + request: Final = FireworksAIConfig().transform_request( + model="accounts/fireworks/models/gpt-oss-120b", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert "cache_control" not in json.dumps(request["messages"]) + assert json.dumps(messages) == caller_messages_before + + # ----------------------------------------------------------------------------- # Regression tests for legacy / OpenAPI $ref defs in tool parameters. # @@ -2477,3 +2498,43 @@ def test_translate_developer_role_follows_the_deployment_model_info_over_the_cos ) assert [message["role"] for message in messages] == expected_roles + + +def test_fireworks_request_rewrites_tools_without_touching_the_callers_tool() -> None: + tools: Final = [ + { + "type": "function", + "function": { + "name": "create_thing", + "strict": True, + "parameters": { + "type": "object", + "properties": {"thing": {"$ref": "#/definitions/Thing"}}, + "definitions": {"Thing": {"type": "object", "properties": {"id": {"type": "string"}}}}, + }, + }, + } + ] + tools_before: Final = copy.deepcopy(tools) + + request: Final = FireworksAIConfig().transform_request( + model="accounts/fireworks/models/gpt-oss-120b", + messages=[{"role": "user", "content": "Make a thing"}], + optional_params={"tools": tools}, + litellm_params={}, + headers={}, + ) + + assert request["tools"] == [ + { + "type": "function", + "function": { + "name": "create_thing", + "parameters": { + "type": "object", + "properties": {"thing": {"type": "object", "properties": {"id": {"type": "string"}}}}, + }, + }, + } + ] + assert tools == tools_before diff --git a/tests/unit/llms/groq/chat/test_groq_chat_transformation.py b/tests/unit/llms/groq/chat/test_groq_chat_transformation.py index 7aa8c65215e..a493a259f39 100644 --- a/tests/unit/llms/groq/chat/test_groq_chat_transformation.py +++ b/tests/unit/llms/groq/chat/test_groq_chat_transformation.py @@ -1,3 +1,4 @@ +import copy import logging from typing import Final from unittest.mock import patch @@ -589,3 +590,19 @@ async def test_streamed_reasoning_reply_can_be_replayed(is_async: bool, with_too } ] assert reply.reasoning_content == "I should check the weather." + + +def test_groq_request_drops_null_assistant_fields_without_touching_the_callers_messages() -> None: + messages: Final = [ + {"role": "user", "content": "Hi"}, + {"role": "assistant", "content": "Hello", "function_call": None, "tool_calls": None}, + {"role": "user", "content": "Again"}, + ] + messages_before: Final = copy.deepcopy(messages) + + body: Final = GroqChatConfig().transform_request( + model="openai/gpt-oss-120b", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert body["messages"][1] == {"role": "assistant", "content": "Hello"} + assert messages == messages_before diff --git a/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py b/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py index c792d5dffcd..fc5ae3631ff 100644 --- a/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py +++ b/tests/unit/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py @@ -1,4 +1,6 @@ +import copy import json +from typing import Final from unittest.mock import MagicMock, patch import pytest @@ -397,3 +399,43 @@ def test_hosted_vllm_custom_tools_use_top_level_input_schema(): assert tools[0]["function"]["name"] == "search" assert tools[0]["function"]["description"] == "Search docs" assert tools[0]["function"]["parameters"] == input_schema + + +def _strict_weather_tool() -> dict[str, object]: + return { + "type": "function", + "function": { + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + "additionalProperties": False, + }, + }, + } + + +def test_hosted_vllm_tools_are_rewritten_without_touching_the_callers_tool() -> None: + tools: Final = [_strict_weather_tool()] + tools_before: Final = copy.deepcopy(tools) + + optional_params: Final = HostedVLLMChatConfig().map_openai_params( + non_default_params={"tools": tools}, optional_params={}, model="gpt-oss-120b", drop_params=False + ) + + assert optional_params["tools"] == [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ] + assert tools == tools_before diff --git a/tests/unit/llms/mistral/test_mistral_chat_transformation.py b/tests/unit/llms/mistral/test_mistral_chat_transformation.py index 720a1b4f458..af18a9caf3d 100644 --- a/tests/unit/llms/mistral/test_mistral_chat_transformation.py +++ b/tests/unit/llms/mistral/test_mistral_chat_transformation.py @@ -1,3 +1,4 @@ +import copy import json import logging from collections.abc import Mapping @@ -1212,7 +1213,10 @@ def test_mistral_sends_only_the_text_items_of_a_native_thinking_chunk_beside_an_ "content": [ { "type": "thinking", - "thinking": [{"type": "text", "text": "It has whiskers."}, {"type": "reference", "reference_ids": [1]}], + "thinking": [ + {"type": "text", "text": "It has whiskers."}, + {"type": "reference", "reference_ids": [1]}, + ], }, {"type": "text", "text": "A cat."}, ], @@ -1672,6 +1676,18 @@ def test_mistral_transform_request_drops_an_empty_tool_message_whose_call_id_is_ assert [m.get("role") for m in request["messages"]] == ["user", "assistant"] +def test_mistral_request_drops_a_user_name_without_touching_the_callers_message() -> None: + messages: Final = [{"role": "user", "content": "Hi"}, {"role": "user", "content": "", "name": "alice"}] + messages_before: Final = copy.deepcopy(messages) + + body: Final = MistralConfig().transform_request( + model="mistral-medium-latest", messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert "name" not in json.dumps(body["messages"]) + assert messages == messages_before + + def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]: return tuple( r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage() diff --git a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py index 5b85a94b24d..79fe7061297 100644 --- a/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py +++ b/tests/unit/llms/openai/chat/test_openai_gpt_transformation.py @@ -1824,6 +1824,94 @@ class TestToolSchemaCombinatorFlatteningForOpenAI: assert request["tools"][0] is tool +class TestCacheControlStrippingDoesNotMutateCallerInput: + """ + Stripping cache_control for a provider that cannot use it must not reach back + into the caller's own message and tool objects. + + filter_value_from_dict deletes the key in place and recurses into nested dicts + and lists, and remove_cache_control_flag_from_messages_and_tools assigned the + result back into the caller's list, so one call to any OpenAI-compatible + provider permanently stripped cache_control from a list the caller still holds. + Reusing that list on Anthropic or Bedrock afterwards then silently lost prompt + caching, with no error and full-price billing. + """ + + def setup_method(self): + self.config = OpenAIGPTConfig() + + @pytest.fixture(autouse=True) + def _clean_openai_base_env(self, monkeypatch): + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None, raising=False) + + @staticmethod + def _messages(): + return [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": "a long cached system prompt", + "cache_control": {"type": "ephemeral"}, + } + ], + }, + { + "role": "user", + "content": "Hello", + "cache_control": {"type": "ephemeral"}, + }, + ] + + @staticmethod + def _tools(): + return [ + { + "type": "function", + "function": {"name": "get_weather", "parameters": {"type": "object"}}, + "cache_control": {"type": "ephemeral"}, + } + ] + + def _transform(self, messages, tools): + return self.config.transform_request( + model="gpt-4o", + messages=messages, + optional_params={"tools": tools}, + litellm_params={"custom_llm_provider": "openai", "api_base": None}, + headers={}, + ) + + def test_caller_messages_and_tools_keep_cache_control(self): + messages, tools = self._messages(), self._tools() + messages_before, tools_before = copy.deepcopy(messages), copy.deepcopy(tools) + + request = self._transform(messages, tools) + + assert "cache_control" not in json.dumps(request) + assert messages == messages_before + assert tools == tools_before + + def test_a_later_anthropic_call_still_sees_cache_control(self): + """The user-visible consequence: the same message list reused on a provider + that does support caching must still carry the cache breakpoints.""" + messages = self._messages() + + self._transform(messages, self._tools()) + + anthropic_body = litellm.AnthropicConfig().transform_request( + model="claude-3-5-sonnet-20240620", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + assert "cache_control" in json.dumps(anthropic_body) + + def _developer_message_after_a_user_turn() -> list[dict[str, str]]: return [ {"role": "system", "content": "You are terse."}, @@ -1916,3 +2004,53 @@ def test_completion_leaves_the_callers_closing_developer_message_unchanged_on_a_ ) assert json.loads(chat.calls.last.request.content)["messages"][-1]["role"] == "user" assert messages == sent + + +def _user_turn_with_litellm_only_media_fields() -> list[dict[str, object]]: + return [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe these"}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png", "format": "image/png"}}, + {"type": "image_url", "image_url": "https://example.com/dog.png"}, + { + "type": "file", + "file": {"file_data": "data:application/pdf;base64,JVBERi0=", "format": "application/pdf"}, + }, + ], + } + ] + + +@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) +async def test_openai_request_rewrites_media_parts_without_touching_the_callers_message( + is_async: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", None, raising=False) + config: Final = OpenAIGPTConfig() + messages: Final = _user_turn_with_litellm_only_media_fields() + messages_before: Final = copy.deepcopy(messages) + request_kwargs: Final = { + "model": "gpt-5.5", + "messages": messages, + "optional_params": {}, + "litellm_params": {"custom_llm_provider": "openai", "api_base": None}, + "headers": {}, + } + + body: Final = ( + await config.async_transform_request(**request_kwargs) + if is_async + else config.transform_request(**request_kwargs) + ) + + assert body["messages"][0]["content"] == [ + {"type": "text", "text": "Describe these"}, + {"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}}, + {"type": "image_url", "image_url": {"url": "https://example.com/dog.png"}}, + {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0=", "filename": "my_file.pdf"}}, + ] + assert messages == messages_before diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index f5a33f87fd0..41c54bf6cc9 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -1,3 +1,4 @@ +import copy import json import logging from collections.abc import Callable, Mapping @@ -541,6 +542,50 @@ class TestOpenAIResponsesAPIConfig: assert "cache_control" not in result["tools"][0] assert result["tools"][0]["name"] == "get_weather" + def test_transform_does_not_strip_cache_control_from_the_callers_input(self): + """Stripping for OpenAI must not reach back into the caller's own objects. + + `filter_value_from_dict` deletes the key in place and recurses, and + `_validate_input_param` passes plain dict items through by reference, so + the caller's input list used to lose its cache breakpoints. Reusing that + list on Anthropic or Bedrock afterwards then silently lost prompt caching. + Same defect as the Chat Completions path. + """ + input_with_cache_control = [ + { + "role": "system", + "content": [ + { + "type": "input_text", + "text": "a long cached system prompt", + "cache_control": {"type": "ephemeral"}, + } + ], + } + ] + tools_with_cache_control = [ + { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object"}, + "cache_control": {"type": "ephemeral"}, + } + ] + input_before = copy.deepcopy(input_with_cache_control) + tools_before = copy.deepcopy(tools_with_cache_control) + + result = self.config.transform_responses_api_request( + model=self.model, + input=input_with_cache_control, + response_api_optional_request_params={"tools": tools_with_cache_control}, + litellm_params={}, + headers={}, + ) + + assert "cache_control" not in json.dumps(result) + assert input_with_cache_control == input_before + assert tools_with_cache_control == tools_before + def test_transform_preserves_input_without_cache_control(self): """Inputs without cache_control must pass through unmodified.""" input_clean = [ @@ -4607,10 +4652,10 @@ def test_cache_control_still_marks_a_part_whose_own_marker_is_malformed_under_dr } -def _caller_input_after_transform(model: str, marker: object) -> str: +def _hinted_caller_input(marker: object) -> list[dict[str, object]]: hint: Final = {"type": "ephemeral"} marked: Final = {} if marker is None else {"prompt_cache_breakpoint": marker} - input: Final = [ + return [ { "type": "message", "role": "user", @@ -4626,6 +4671,10 @@ def _caller_input_after_transform(model: str, marker: object) -> str: "content": [{"type": "output_text", "text": "ok", "cache_control": hint, **marked}], }, ] + + +def _caller_input_after_transform(model: str, marker: object) -> str: + input: Final = _hinted_caller_input(marker) _CONFIGS[0].transform_responses_api_request( model=model, input=cast("ResponseInputParam", input), # cast-ok: plain dicts shaped like the SDK's input items @@ -4647,6 +4696,15 @@ def test_a_marker_leaves_the_caller_request_that_fallbacks_and_logs_read_as_an_u assert _caller_input_after_transform(model, marker).replace(marker_json, "") == unmarked +@pytest.mark.parametrize("model", ("gpt-4.1", "gpt-6.1-sol")) +@pytest.mark.parametrize("marker", (None, _VALID_MARKERS[0], "yes"), ids=("unmarked", "explicit", "malformed")) +def test_the_caller_input_keeps_its_cache_control_hints_for_a_fallback_deployment( + monkeypatch: pytest.MonkeyPatch, model: str, marker: object +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + assert _caller_input_after_transform(model, marker) == json.dumps(_hinted_caller_input(marker)) + + @pytest.mark.asyncio @pytest.mark.usefixtures(httpx_transport.__name__) async def test_aresponses_forwards_previous_response_id_to_openai() -> None: diff --git a/tests/unit/llms/openai/test_o_series_transformation.py b/tests/unit/llms/openai/test_o_series_transformation.py index e78e1a6f12b..9d39dfda60a 100644 --- a/tests/unit/llms/openai/test_o_series_transformation.py +++ b/tests/unit/llms/openai/test_o_series_transformation.py @@ -1,3 +1,4 @@ +import copy from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -198,3 +199,23 @@ def test_translate_developer_role_keeps_developer_messages_untouched() -> None: ) == messages ) + + +def test_o_series_request_turns_system_into_user_without_touching_the_callers_messages( + monkeypatch: pytest.MonkeyPatch, +) -> None: + model: Final = "o-series-test-model-without-system-role" + monkeypatch.setitem( + litellm.model_cost, + model, + {"litellm_provider": "openai", "mode": "chat", "supports_system_messages": False}, + ) + messages: Final = [{"role": "system", "content": "Be terse"}, {"role": "user", "content": "Hi"}] + messages_before: Final = copy.deepcopy(messages) + + body: Final = OpenAIOSeriesConfig().transform_request( + model=model, messages=messages, optional_params={}, litellm_params={}, headers={} + ) + + assert [message["role"] for message in body["messages"]] == ["user", "user"] + assert messages == messages_before diff --git a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py index 3d659641f54..6cef65f8a88 100644 --- a/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py +++ b/tests/unit/llms/openrouter/chat/test_openrouter_chat_transformation.py @@ -1,5 +1,8 @@ +import copy import json +from collections.abc import Iterator +from itertools import chain from typing import Final from unittest.mock import MagicMock @@ -874,6 +877,58 @@ def test_openrouter_streaming_reasoning_is_preserved() -> None: assert assembled.choices[0].message.reasoning_content == "Consider the evidence. Apply the rule." +def _five_message_level_cache_hints() -> list[dict[str, object]]: + return [ + {"role": "system", "content": "system prompt", "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "turn 1", "cache_control": {"type": "ephemeral"}}, + {"role": "assistant", "content": "answer 1", "cache_control": {"type": "ephemeral"}}, + {"role": "user", "content": "turn 2", "cache_control": {"type": "ephemeral"}}, + {"role": "assistant", "content": "answer 2"}, + {"role": "user", "content": "turn 3", "cache_control": {"type": "ephemeral"}}, + ] + + +def _cached_texts(node: object) -> Iterator[str]: + if isinstance(node, dict): + yield from ((node["text"],) if "cache_control" in node and "text" in node else ()) + yield from chain.from_iterable(map(_cached_texts, node.values())) + elif isinstance(node, list): + yield from chain.from_iterable(map(_cached_texts, node)) + + +@pytest.mark.parametrize( + ("drop_params", "expected_cached_texts"), + [ + (True, ["answer 1", "turn 1", "turn 2", "turn 3"]), + ("true", ["answer 1", "turn 1", "turn 2", "turn 3"]), + (False, ["answer 1", "system prompt", "turn 1", "turn 2", "turn 3"]), + ], + ids=[ + "drop_params_keeps_the_newest_four", + "string_drop_params_keeps_the_newest_four", + "without_drop_params_all_five_pass_through", + ], +) +def test_openrouter_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params( + drop_params: bool | str, expected_cached_texts: list[str], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + messages: Final = _five_message_level_cache_hints() + messages_before: Final = copy.deepcopy(messages) + + body: Final = OpenrouterConfig().transform_request( + model="anthropic/claude-haiku-5.5", + messages=messages, + optional_params={}, + litellm_params={"drop_params": drop_params, "custom_llm_provider": "openrouter"}, + headers={}, + ) + + assert sorted(_cached_texts(body["messages"])) == expected_cached_texts + assert json.dumps(body).count('"cache_control"') == len(expected_cached_texts) + assert messages == messages_before + + def _transform_openrouter_response(body: dict[str, object]) -> litellm.ModelResponse: return OpenrouterConfig().transform_response( model="openrouter/openai/gpt-4o-mini", diff --git a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py index 78b9e930001..bc8421c80f8 100644 --- a/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py +++ b/tests/unit/llms/vertex_ai/context_caching/test_vertex_ai_context_caching.py @@ -1,3 +1,4 @@ +import json from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch @@ -2507,3 +2508,46 @@ def test_check_and_create_cache_clamps_negative_creation_tokens() -> None: assert cache_name == "cachedContents/created" assert logging_obj.vertex_ai_cache_creation_input_tokens == 0 + + +def _recording_gemini_client(calls: list[httpx.Request]) -> HTTPHandler: + def route(request: httpx.Request) -> httpx.Response: + calls.append(request) + if request.url.path.endswith("/cachedContents"): + return httpx.Response(200, json={} if request.method == "GET" else _created_cache_body(3000)) + return httpx.Response( + 200, + json={ + "candidates": [{"content": {"role": "model", "parts": [{"text": "OK"}]}, "finishReason": "STOP"}], + "usageMetadata": {"promptTokenCount": 3000, "candidatesTokenCount": 1, "totalTokenCount": 3001}, + }, + ) + + return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(route))) + + +def test_a_plain_system_prompt_goes_into_the_gemini_cache_instead_of_beside_it(local_model_cost_map) -> None: + calls: Final[list[httpx.Request]] = [] + litellm.completion( + model="gemini/gemini-flash-latest", + messages=[ + {"role": "system", "content": "You are terse."}, + { + "role": "user", + "content": [ + {"type": "text", "text": "plain system prefix " * 3000, "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "Reply with OK only."}, + ], + }, + ], + api_key="test-key", + client=_recording_gemini_client(calls), + ) + + created: Final = next( + json.loads(call.content) for call in calls if call.method == "POST" and call.url.path.endswith("/cachedContents") + ) + generated: Final = next(json.loads(call.content) for call in calls if call.url.path.endswith(":generateContent")) + assert created["system_instruction"] == {"parts": [{"text": "You are terse."}]} + assert generated["cachedContent"] == "cachedContents/created" + assert "system_instruction" not in generated diff --git a/tests/unit/llms/watsonx/test_watsonx.py b/tests/unit/llms/watsonx/test_watsonx.py index 4afa717debe..31d2875d2b9 100644 --- a/tests/unit/llms/watsonx/test_watsonx.py +++ b/tests/unit/llms/watsonx/test_watsonx.py @@ -1,4 +1,5 @@ import asyncio +import copy import importlib import json from typing import Final, Optional, cast @@ -10,6 +11,7 @@ import pytest import litellm from litellm import completion, embedding from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.watsonx.chat.transformation import IBMWatsonXChatConfig from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -542,3 +544,43 @@ def test_watsonx_deployment_space_id_embedding(monkeypatch, watsonx_embedding_ca # nor space_id or project_id is required by wx.ai API when inferencing deployment assert "project_id" not in json_data and "space_id" not in json_data + + +def _strict_weather_tool() -> dict[str, object]: + return { + "type": "function", + "function": { + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + "additionalProperties": False, + }, + }, + } + + +def test_watsonx_chat_tools_are_rewritten_without_touching_the_callers_tool() -> None: + tools: Final = [_strict_weather_tool()] + tools_before: Final = copy.deepcopy(tools) + + optional_params: Final = IBMWatsonXChatConfig().map_openai_params( + non_default_params={"tools": tools}, optional_params={}, model="ibm/granite-4-h-small", drop_params=False + ) + + assert optional_params["tools"] == [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ] + assert tools == tools_before diff --git a/tests/unit/llms/xai/test_xai_chat_transformation.py b/tests/unit/llms/xai/test_xai_chat_transformation.py index 69e620dbc8a..37086f797fa 100644 --- a/tests/unit/llms/xai/test_xai_chat_transformation.py +++ b/tests/unit/llms/xai/test_xai_chat_transformation.py @@ -1,3 +1,4 @@ +import copy import json import os from typing import Final @@ -581,3 +582,44 @@ def test_xai_grok_4_frequency_penalty_not_supported(model): """ supported_params = XAIChatConfig().get_supported_openai_params(model=model) assert "frequency_penalty" not in supported_params + + +def _strict_weather_tool() -> dict[str, object]: + return { + "type": "function", + "function": { + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + "additionalProperties": False, + }, + }, + } + + +def test_xai_tools_are_rewritten_without_touching_the_callers_tool() -> None: + tools: Final = [_strict_weather_tool()] + tools_before: Final = copy.deepcopy(tools) + + optional_params: Final = XAIChatConfig().map_openai_params( + non_default_params={"tools": tools}, optional_params={}, model="grok-4.3", drop_params=False + ) + + assert optional_params["tools"] == [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + "additionalProperties": False, + }, + }, + } + ] + assert tools == tools_before diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 4250c78c669..b1047a345d7 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -6314,6 +6314,17 @@ def test_marks_past_the_provider_cap_keep_the_newest_under_drop_params(drop_para assert messages[1] == {"role": "user", "content": "turn 0"} +def test_marks_past_the_provider_cap_inside_one_message_keep_the_newest_under_drop_params(): + parts: Final = tuple( + {"type": "input_text", "text": f"part {index}", "cache_control": _EPHEMERAL} for index in range(1, 6) + ) + + messages: Final = _bridged_request({"role": "user", "content": list(parts)}, drop_params=True) + + content: Final = TypeAdapter(list[dict[str, object]]).validate_python(messages[-1]["content"]) + assert ["cache_control" in part for part in content] == [False, True, True, True, True] + + def test_global_drop_params_keeps_the_newest_marks(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr(litellm, "drop_params", True)