mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge remote-tracking branch 'origin/main' into litellm_fix_claude_stop_reasons_pause_turn_refusal
# Conflicts: # litellm/llms/anthropic/chat/transformation.py
This commit is contained in:
commit
a1b098f4c4
67 changed files with 3023 additions and 328 deletions
|
|
@ -634,6 +634,8 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60
|
|||
BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024)
|
||||
# Anthropic's Messages API rejects thinking.budget_tokens < 1024.
|
||||
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024
|
||||
ANTHROPIC_MAX_USER_ID_LENGTH: Final = 512
|
||||
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH: Final = 255
|
||||
ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:"
|
||||
REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5)
|
||||
DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000)
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ import itertools
|
|||
import os
|
||||
import re
|
||||
from collections.abc import Callable, Container, Iterable, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
|
@ -102,6 +102,192 @@ def _validated_object_list(value: object) -> list[object] | None:
|
|||
return None
|
||||
|
||||
|
||||
_MessageT: Final = TypeVar("_MessageT")
|
||||
_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None]
|
||||
_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls")
|
||||
|
||||
|
||||
def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]:
|
||||
lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS)
|
||||
return tuple(
|
||||
itertools.chain.from_iterable(
|
||||
_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _block_carries_cache_control(block: object) -> bool:
|
||||
return isinstance(block, dict) and _CONTENT_PART_ADAPTER.validate_python(block).get("cache_control") is not None
|
||||
|
||||
|
||||
_ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT: Final = frozenset({"tool", "function"})
|
||||
|
||||
|
||||
def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool:
|
||||
if fields.get("cache_control") is None:
|
||||
return False
|
||||
return (
|
||||
not isinstance(fields.get("content"), list)
|
||||
or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
|
||||
)
|
||||
|
||||
|
||||
def cache_control_count(message: object) -> int:
|
||||
if not isinstance(message, dict):
|
||||
return 0
|
||||
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
|
||||
block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block))
|
||||
return int(_message_mark_reaches_provider(fields)) + block_marks
|
||||
|
||||
|
||||
def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]:
|
||||
others: Final = {key: value for key, value in fields.items() if key != "cache_control"}
|
||||
if "cache_control" not in fields:
|
||||
return others
|
||||
cache_control: Final = rewrite(fields["cache_control"])
|
||||
return others if cache_control is None else {**others, "cache_control": cache_control}
|
||||
|
||||
|
||||
def block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object:
|
||||
if not isinstance(block, dict):
|
||||
return block
|
||||
return _fields_with_cache_control(_CONTENT_PART_ADAPTER.validate_python(block), rewrite)
|
||||
|
||||
|
||||
def with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT:
|
||||
if not isinstance(message, dict):
|
||||
return message
|
||||
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
|
||||
blocks_rewritten: Final = {
|
||||
key: (
|
||||
[block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)]
|
||||
if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list)
|
||||
else value
|
||||
)
|
||||
for key, value in fields.items()
|
||||
}
|
||||
return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped
|
||||
_MessageT, _fields_with_cache_control(blocks_rewritten, rewrite)
|
||||
)
|
||||
|
||||
|
||||
def dropped_cache_control(_: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _first_marked_index(value: object) -> int | None:
|
||||
if not isinstance(value, list):
|
||||
return None
|
||||
blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
|
||||
return next((index for index, block in enumerate(blocks) if _block_carries_cache_control(block)), None)
|
||||
|
||||
|
||||
_MarkSlot: TypeAlias = tuple[str, int]
|
||||
_MESSAGE_MARK_SLOT: Final[_MarkSlot] = ("cache_control", -1)
|
||||
|
||||
|
||||
def _block_mark_slots(fields: Mapping[str, object], key: str) -> tuple[_MarkSlot, ...]:
|
||||
value: Final = fields.get(key)
|
||||
if not isinstance(value, list):
|
||||
return ()
|
||||
blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
|
||||
return tuple((key, index) for index, block in enumerate(blocks) if _block_carries_cache_control(block))
|
||||
|
||||
|
||||
def _mark_slots_in_wire_order(fields: Mapping[str, object]) -> tuple[_MarkSlot, ...]:
|
||||
content_slots: Final = _block_mark_slots(fields, "content")
|
||||
tool_call_slots: Final = _block_mark_slots(fields, "tool_calls")
|
||||
if not _message_mark_reaches_provider(fields):
|
||||
return content_slots + tool_call_slots
|
||||
if isinstance(fields.get("content"), list):
|
||||
return content_slots + (_MESSAGE_MARK_SLOT,) + tool_call_slots
|
||||
return (_MESSAGE_MARK_SLOT,) + content_slots + tool_call_slots
|
||||
|
||||
|
||||
def _blocks_without_marks(key: str, value: object, dropped_slots: frozenset[_MarkSlot]) -> object:
|
||||
if key not in _MARK_CARRYING_MESSAGE_FIELDS or not isinstance(value, list):
|
||||
return value
|
||||
return [
|
||||
block_with_cache_control(block, dropped_cache_control) if (key, index) in dropped_slots else block
|
||||
for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(value))
|
||||
]
|
||||
|
||||
|
||||
def _without_oldest_marks(message: _MessageT, dropped: int) -> _MessageT:
|
||||
if dropped == 0 or not isinstance(message, dict):
|
||||
return message
|
||||
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
|
||||
dropped_slots: Final = frozenset(_mark_slots_in_wire_order(fields)[:dropped])
|
||||
blocks_unmarked: Final = {key: _blocks_without_marks(key, value, dropped_slots) for key, value in fields.items()}
|
||||
return cast( # cast-ok: the same message with its oldest wire-order marks dropped
|
||||
_MessageT,
|
||||
_fields_with_cache_control(blocks_unmarked, dropped_cache_control)
|
||||
if _MESSAGE_MARK_SLOT in dropped_slots
|
||||
else blocks_unmarked,
|
||||
)
|
||||
|
||||
|
||||
def tool_message_with_single_cache_mark(message: _MessageT) -> _MessageT:
|
||||
if not isinstance(message, dict):
|
||||
return message
|
||||
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
|
||||
content: Final = fields.get("content")
|
||||
if fields.get("role") != "tool" or not isinstance(content, list):
|
||||
return message
|
||||
kept_index: Final = None if fields.get("cache_control") is not None else _first_marked_index(content)
|
||||
return cast( # cast-ok: the same tool message keeping only the mark its single cache point reads
|
||||
_MessageT,
|
||||
{
|
||||
**fields,
|
||||
"content": [
|
||||
block if index == kept_index else block_with_cache_control(block, dropped_cache_control)
|
||||
for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(content))
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
_BudgetSpend: TypeAlias = Callable[[tuple[int, int], int], tuple[int, int]]
|
||||
|
||||
|
||||
def _newest_marks_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
|
||||
remaining, _ = state
|
||||
kept: Final = min(marks, remaining)
|
||||
return remaining - kept, marks - kept
|
||||
|
||||
|
||||
def _whole_message_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
|
||||
remaining, _ = state
|
||||
if marks > remaining:
|
||||
return remaining, marks
|
||||
return remaining - marks, 0
|
||||
|
||||
|
||||
def _messages_within_budget(
|
||||
messages: Sequence[_MessageT], reserved_marks: int, spend: _BudgetSpend
|
||||
) -> Sequence[_MessageT]:
|
||||
budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0)
|
||||
newest_first: Final = tuple(reversed(messages))
|
||||
budget_after_each: Final = itertools.accumulate(
|
||||
(cache_control_count(message) for message in newest_first), spend, initial=(budget, 0)
|
||||
)
|
||||
dropped_counts: Final = tuple(dropped for _, dropped in itertools.islice(budget_after_each, 1, None))
|
||||
kept_newest_first: Final = tuple(
|
||||
_without_oldest_marks(message, dropped) for message, dropped in zip(newest_first, dropped_counts)
|
||||
)
|
||||
return tuple(reversed(kept_newest_first))
|
||||
|
||||
|
||||
def newest_marks_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]:
|
||||
return _messages_within_budget(messages, reserved_marks, _newest_marks_kept)
|
||||
|
||||
|
||||
def whole_messages_within_cache_control_limit(
|
||||
messages: Sequence[_MessageT], reserved_marks: int
|
||||
) -> Sequence[_MessageT]:
|
||||
return _messages_within_budget(messages, reserved_marks, _whole_message_kept)
|
||||
|
||||
|
||||
def _part_with_cache_control_renamed(part: object) -> object:
|
||||
if not _accepts_prompt_cache_breakpoint(part):
|
||||
return part
|
||||
|
|
@ -1439,3 +1625,25 @@ class AnthropicCacheControlHook(CustomPromptManagement):
|
|||
llm_router=None,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def messages_within_cache_control_cap(
|
||||
messages: Sequence[_MessageT], tools: object, cache_control: object = None, request_kwargs: object = None
|
||||
) -> Sequence[_MessageT]:
|
||||
"""Under drop_params, drop the oldest message hints the provider's cache_control cap has no room for.
|
||||
|
||||
Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first.
|
||||
"""
|
||||
reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints(
|
||||
_validated_object_list(tools), cache_control, request_kwargs
|
||||
)
|
||||
kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks)
|
||||
dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept))
|
||||
if dropped > 0:
|
||||
verbose_logger.warning(
|
||||
"Dropped %d cache_control hint(s) from the oldest messages because drop_params is on and the "
|
||||
"provider accepts at most %d cache_control blocks per request",
|
||||
dropped,
|
||||
MAX_CACHE_CONTROL_BLOCKS,
|
||||
)
|
||||
return kept
|
||||
|
|
|
|||
|
|
@ -208,11 +208,14 @@ class StandardBuiltInToolCostTracking:
|
|||
usage=usage, response_object=response_object
|
||||
)
|
||||
|
||||
response_model: Final = _output_item_field(response_object, "model")
|
||||
if model_info is not None and resolved_usage is not None and resolved_provider is not None:
|
||||
result: Final = get_cost_for_web_search_request(
|
||||
custom_llm_provider=resolved_provider,
|
||||
usage=resolved_usage,
|
||||
model_info=model_info,
|
||||
response_model=response_model if isinstance(response_model, str) else None,
|
||||
lookup_model_info=StandardBuiltInToolCostTracking._safe_get_model_info,
|
||||
)
|
||||
if result is not None:
|
||||
return result
|
||||
|
|
|
|||
|
|
@ -88,17 +88,22 @@ def handle_any_messages_to_chat_completion_str_messages_conversion(
|
|||
return [{"input": str(messages)}]
|
||||
|
||||
|
||||
def _message_with_str_content(message: AllMessageValues) -> AllMessageValues:
|
||||
texts: Final = convert_content_list_to_str(message=message)
|
||||
if not texts:
|
||||
return message
|
||||
return cast( # cast-ok: the same message with its content flattened to one string
|
||||
AllMessageValues, {**message, "content": texts}
|
||||
)
|
||||
|
||||
|
||||
def handle_messages_with_content_list_to_str_conversion(
|
||||
messages: list[AllMessageValues],
|
||||
) -> list[AllMessageValues]:
|
||||
"""
|
||||
Handles messages with content list conversion
|
||||
Returns new messages with each content list flattened to a string, leaving the caller's messages untouched
|
||||
"""
|
||||
for message in messages:
|
||||
texts = convert_content_list_to_str(message=message)
|
||||
if texts:
|
||||
message["content"] = texts
|
||||
return messages
|
||||
return [_message_with_str_content(message) for message in messages]
|
||||
|
||||
|
||||
def strip_name_from_message(message: AllMessageValues, allowed_name_roles: list[str] = ["user"]) -> AllMessageValues:
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ from litellm.types.utils import CallTypes
|
|||
from . import *
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Callable
|
||||
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import (
|
||||
BaseTranslation,
|
||||
)
|
||||
|
|
@ -30,7 +32,29 @@ def get_cost_for_google_maps_grounding_request(
|
|||
return cost_per_google_maps_grounding_request(usage=usage, model_info=model_info)
|
||||
|
||||
|
||||
def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", model_info: "ModelInfo") -> float | None:
|
||||
def _with_served_model_search_pricing(
|
||||
model_info: "ModelInfo",
|
||||
response_model: str | None,
|
||||
served_model_provider: str,
|
||||
lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None",
|
||||
) -> "ModelInfo":
|
||||
if model_info.get("search_context_cost_per_query") is not None or response_model is None:
|
||||
return model_info
|
||||
served_model_info: Final = (
|
||||
lookup_model_info(response_model, served_model_provider) if lookup_model_info is not None else None
|
||||
)
|
||||
if served_model_info is None:
|
||||
return model_info
|
||||
return {**model_info, "search_context_cost_per_query": served_model_info.get("search_context_cost_per_query")}
|
||||
|
||||
|
||||
def get_cost_for_web_search_request(
|
||||
custom_llm_provider: str,
|
||||
usage: "Usage",
|
||||
model_info: "ModelInfo",
|
||||
response_model: str | None = None,
|
||||
lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None" = None,
|
||||
) -> float | None:
|
||||
"""
|
||||
Get the cost for a web search request for a given model.
|
||||
|
||||
|
|
@ -38,6 +62,8 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
|
|||
custom_llm_provider: The custom LLM provider.
|
||||
usage: The usage object.
|
||||
model_info: The model info.
|
||||
response_model: The model the provider response names.
|
||||
lookup_model_info: Looks up a model's cost map entry, returning None when it is not mapped.
|
||||
"""
|
||||
if custom_llm_provider == "gemini":
|
||||
from .gemini.cost_calculator import cost_per_web_search_request
|
||||
|
|
@ -46,17 +72,25 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
|
|||
elif custom_llm_provider == "anthropic":
|
||||
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
|
||||
|
||||
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
|
||||
return get_cost_for_anthropic_web_search(
|
||||
model_info=_with_served_model_search_pricing(model_info, response_model, "anthropic", lookup_model_info),
|
||||
usage=usage,
|
||||
)
|
||||
elif custom_llm_provider.startswith("vertex_ai"):
|
||||
# Anthropic Claude models on Vertex AI populate server_tool_use.web_search_requests
|
||||
# (same as the direct Anthropic API), not prompt_tokens_details.web_search_requests
|
||||
# (which is the Gemini field). Route claude-* models to the Anthropic calculator.
|
||||
model_key: Final[str] = model_info.get("key", "") if model_info else ""
|
||||
if "claude" in model_key.lower():
|
||||
if "claude" in (response_model or model_key).lower():
|
||||
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
|
||||
|
||||
verbose_logger.debug("vertex_ai/claude model detected — routing web search cost to Anthropic calculator")
|
||||
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
|
||||
return get_cost_for_anthropic_web_search(
|
||||
model_info=_with_served_model_search_pricing(
|
||||
model_info, response_model, "vertex_ai", lookup_model_info
|
||||
),
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
from .vertex_ai.gemini.cost_calculator import (
|
||||
cost_per_web_search_request as cost_per_web_search_request_vertex_ai,
|
||||
|
|
@ -71,7 +105,10 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
|
|||
return None
|
||||
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
|
||||
|
||||
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
|
||||
return get_cost_for_anthropic_web_search(
|
||||
model_info=_with_served_model_search_pricing(model_info, response_model, "azure_ai", lookup_model_info),
|
||||
usage=usage,
|
||||
)
|
||||
elif custom_llm_provider == "perplexity":
|
||||
# Perplexity handles search costs internally in its own cost calculator
|
||||
# Return 0.0 to indicate costs are already accounted for
|
||||
|
|
|
|||
|
|
@ -170,6 +170,7 @@ _TOOLS_ADAPTER: Final = TypeAdapter(list[object])
|
|||
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_RESPONSE_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
|
||||
_PENDING_WEB_SEARCH_RESULT: Final = MappingProxyType({"content": ()})
|
||||
_REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
|
||||
("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
|
||||
)
|
||||
|
|
@ -2257,6 +2258,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
headers=headers,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _messages_within_cache_control_cap(
|
||||
messages: Sequence[AllMessageValues], optional_params: Mapping[str, object]
|
||||
) -> Sequence[AllMessageValues]:
|
||||
from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
|
||||
|
||||
return messages_within_cache_control_cap(
|
||||
messages, optional_params.get("tools"), optional_params.get("cache_control"), optional_params
|
||||
)
|
||||
|
||||
def transform_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -2274,9 +2285,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
anthropic_messages_pt,
|
||||
)
|
||||
|
||||
drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True
|
||||
drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
|
||||
capped_messages: Final = (
|
||||
list(
|
||||
self._messages_within_cache_control_cap(
|
||||
messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params)
|
||||
)
|
||||
)
|
||||
if drop_params
|
||||
else messages
|
||||
)
|
||||
|
||||
if "tools" not in optional_params and messages is not None and has_tool_call_blocks(messages):
|
||||
if "tools" not in optional_params and has_tool_call_blocks(capped_messages):
|
||||
optional_params["tools"], _ = self._map_tools(add_dummy_tool(custom_llm_provider="anthropic"))
|
||||
|
||||
# Drop thinking param if thinking is enabled but thinking_blocks are missing
|
||||
|
|
@ -2288,9 +2308,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
# Related issue: https://github.com/BerriAI/litellm/issues/18926
|
||||
if (
|
||||
optional_params.get("thinking") is not None
|
||||
and messages is not None
|
||||
and last_assistant_with_tool_calls_has_no_thinking_blocks(messages)
|
||||
and not any_assistant_message_has_thinking_blocks(messages)
|
||||
and last_assistant_with_tool_calls_has_no_thinking_blocks(capped_messages)
|
||||
and not any_assistant_message_has_thinking_blocks(capped_messages)
|
||||
):
|
||||
if litellm.modify_params:
|
||||
optional_params.pop("thinking", None)
|
||||
|
|
@ -2335,15 +2354,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
_name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request(
|
||||
optional_params=optional_params,
|
||||
)
|
||||
if _name_forward_map:
|
||||
messages = self._rewrite_tool_names_in_messages(messages, _name_forward_map)
|
||||
request_messages: Final = (
|
||||
self._rewrite_tool_names_in_messages(capped_messages, _name_forward_map)
|
||||
if _name_forward_map
|
||||
else capped_messages
|
||||
)
|
||||
if _name_reverse_map and isinstance(litellm_params, dict):
|
||||
litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map
|
||||
|
||||
# Only the leading system run becomes the top-level system prompt. A later
|
||||
# system message stays in the conversation: hoisting it rewrites the cached
|
||||
# prefix and re-bills the whole history at cache-write pricing (#36559).
|
||||
leading_system_run, later_messages = split_leading_system_run(messages)
|
||||
leading_system_run, later_messages = split_leading_system_run(request_messages)
|
||||
anthropic_system_message_list: Final = self.translate_system_message(messages=list(leading_system_run))
|
||||
# Handling anthropic API Prompt Caching
|
||||
if len(anthropic_system_message_list) > 0:
|
||||
|
|
@ -2365,7 +2387,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
|
|||
except Exception as e:
|
||||
raise AnthropicError(
|
||||
status_code=400,
|
||||
message=f"{e}\nReceived Messages={messages}",
|
||||
message=f"{e}\nReceived Messages={request_messages}",
|
||||
) # don't use verbose_logger.exception, if exception is raised
|
||||
|
||||
self.update_headers_with_optional_anthropic_beta(
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ This file contains common utils for anthropic calls.
|
|||
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Container, Mapping, MutableMapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
|
|
@ -19,6 +20,8 @@ from typing_extensions import ReadOnly, TypedDict
|
|||
|
||||
import litellm
|
||||
from litellm.constants import (
|
||||
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH,
|
||||
ANTHROPIC_MAX_USER_ID_LENGTH,
|
||||
DEFAULT_MODEL_CREATED_AT_TIME,
|
||||
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
|
||||
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
|
||||
|
|
@ -178,6 +181,40 @@ def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: boo
|
|||
return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools]
|
||||
|
||||
|
||||
def anthropic_pass_through_api_base() -> str:
|
||||
return os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
|
||||
|
||||
|
||||
def _same_host_and_port(url: httpx.URL, other: str) -> bool:
|
||||
try:
|
||||
other_url: Final = httpx.URL(other)
|
||||
except httpx.InvalidURL:
|
||||
return False
|
||||
return (url.host, url.port) == (other_url.host, other_url.port)
|
||||
|
||||
|
||||
def anthropic_upstream_user_id_limit(url: str) -> int | None:
|
||||
upstream: Final = httpx.URL(url)
|
||||
if not upstream.path.removesuffix("/").endswith("/v1/messages"):
|
||||
return None
|
||||
if upstream.host == "api.anthropic.com":
|
||||
return ANTHROPIC_MAX_USER_ID_LENGTH
|
||||
return (
|
||||
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH
|
||||
if _same_host_and_port(upstream, anthropic_pass_through_api_base())
|
||||
else None
|
||||
)
|
||||
|
||||
|
||||
def _has_email_shape(token: str) -> bool:
|
||||
at_sign: Final = token.find("@", 1)
|
||||
return at_sign != -1 and token.find(".", at_sign + 2, -1) != -1
|
||||
|
||||
|
||||
def anthropic_accepts_user_id(user_id: str, max_length: int) -> bool:
|
||||
return len(user_id) <= max_length and user_id.isprintable() and not any(map(_has_email_shape, user_id.split()))
|
||||
|
||||
|
||||
def is_anthropic_messages_url(url: str) -> bool:
|
||||
"""Check whether a URL addresses Anthropic's Messages API."""
|
||||
parsed_url: Final = urlparse(url)
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from litellm.constants import (
|
|||
)
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
as_str_mapping,
|
||||
drops_unsupported_params,
|
||||
filter_exceptions_from_params,
|
||||
filter_internal_params,
|
||||
map_finish_reason,
|
||||
|
|
@ -2129,6 +2130,24 @@ class AmazonConverseConfig(BaseConfig):
|
|||
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def _claude_messages_within_cache_control_cap(
|
||||
messages: Sequence[AllMessageValues],
|
||||
capability_model: str,
|
||||
optional_params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> Sequence[AllMessageValues]:
|
||||
if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower():
|
||||
return messages
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
messages_within_cache_control_cap,
|
||||
tool_message_with_single_cache_mark,
|
||||
)
|
||||
|
||||
return messages_within_cache_control_cap(
|
||||
tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools")
|
||||
)
|
||||
|
||||
async def _async_transform_request(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -2140,7 +2159,13 @@ class AmazonConverseConfig(BaseConfig):
|
|||
base_model: Final = litellm_params.get("base_model")
|
||||
capability_model: Final = bedrock_capability_model(model, base_model)
|
||||
messages, system_content_blocks = self._transform_system_message(
|
||||
messages, model=capability_model, deployment_model=model
|
||||
list(
|
||||
self._claude_messages_within_cache_control_cap(
|
||||
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
|
||||
)
|
||||
),
|
||||
model=capability_model,
|
||||
deployment_model=model,
|
||||
)
|
||||
|
||||
# Convert last user message to guarded_text if guardrailConfig is present
|
||||
|
|
@ -2221,7 +2246,13 @@ class AmazonConverseConfig(BaseConfig):
|
|||
base_model: Final = litellm_params.get("base_model")
|
||||
capability_model: Final = bedrock_capability_model(model, base_model)
|
||||
messages, system_content_blocks = self._transform_system_message(
|
||||
messages, model=capability_model, deployment_model=model
|
||||
list(
|
||||
self._claude_messages_within_cache_control_cap(
|
||||
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
|
||||
)
|
||||
),
|
||||
model=capability_model,
|
||||
deployment_model=model,
|
||||
)
|
||||
|
||||
# Convert last user message to guarded_text if guardrailConfig is present
|
||||
|
|
|
|||
|
|
@ -130,7 +130,8 @@ class DeepInfraConfig(OpenAIGPTConfig):
|
|||
- Input: {"role": "tool", "content": [{"type": "text", "text": "result"}]}
|
||||
- Output: {"role": "tool", "content": "[{\"type\": \"text\", \"text\": \"result\"}]"}
|
||||
"""
|
||||
for message in messages:
|
||||
copied_messages: Final = [message.copy() for message in messages]
|
||||
for message in copied_messages:
|
||||
if message.get("role") == "tool":
|
||||
content = message.get("content")
|
||||
|
||||
|
|
@ -149,7 +150,7 @@ class DeepInfraConfig(OpenAIGPTConfig):
|
|||
# For complex content, serialize the entire array as JSON string
|
||||
message["content"] = json.dumps(content)
|
||||
|
||||
return messages
|
||||
return copied_messages
|
||||
|
||||
@overload
|
||||
def _transform_messages(
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from litellm.secret_managers.main import get_secret_str
|
|||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
ChatCompletionToolParam,
|
||||
ChatCompletionToolParamFunctionChunk,
|
||||
OpenAIChatCompletionToolParam,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
|
|
@ -45,6 +46,7 @@ from ...openai.chat.gpt_transformation import (
|
|||
OpenAIChatCompletionStreamingHandler,
|
||||
OpenAIGPTConfig,
|
||||
)
|
||||
from ...openai.common_utils import without_cache_control
|
||||
from ..common_utils import (
|
||||
FIREROUTER,
|
||||
FireworksAIException,
|
||||
|
|
@ -480,15 +482,23 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
return (("response_format", _json_schema_response_format(choice_schema, "choice")),)
|
||||
|
||||
def _transform_tools(self, tools: list[OpenAIChatCompletionToolParam]) -> list[OpenAIChatCompletionToolParam]:
|
||||
for tool in tools:
|
||||
if tool.get("type") != "function":
|
||||
continue
|
||||
function = tool["function"]
|
||||
function.pop("strict", None)
|
||||
params = function.get("parameters")
|
||||
if isinstance(params, dict):
|
||||
unpack_legacy_defs(params)
|
||||
return tools
|
||||
return [self._fireworks_tool(tool) for tool in tools]
|
||||
|
||||
@staticmethod
|
||||
def _fireworks_tool(tool: OpenAIChatCompletionToolParam) -> OpenAIChatCompletionToolParam:
|
||||
if tool.get("type") != "function":
|
||||
return tool
|
||||
function: Final = tool["function"]
|
||||
params: Final = function.get("parameters")
|
||||
unstrict_function: Final = cast( # cast-ok: the same function definition minus `strict`
|
||||
ChatCompletionToolParamFunctionChunk, {key: value for key, value in function.items() if key != "strict"}
|
||||
)
|
||||
new_function: Final[ChatCompletionToolParamFunctionChunk] = (
|
||||
{**unstrict_function, "parameters": unpack_legacy_defs(params, copy=True)}
|
||||
if isinstance(params, dict)
|
||||
else unstrict_function
|
||||
)
|
||||
return {**tool, "function": new_function}
|
||||
|
||||
def _transform_messages_helper(
|
||||
self, messages: list[AllMessageValues], model: str, litellm_params: dict
|
||||
|
|
@ -496,10 +506,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
"""
|
||||
Strip fields not permitted by FireworksAI from messages.
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
filter_value_from_dict,
|
||||
)
|
||||
|
||||
supports_vision_value: Final = self._get_model_cost_capability_exact(model=model, capability="supports_vision")
|
||||
for message in messages:
|
||||
if message["role"] == "user":
|
||||
|
|
@ -530,9 +536,11 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
|
|||
model=model,
|
||||
llm_provider="fireworks_ai",
|
||||
)
|
||||
filter_value_from_dict(cast(dict, message), "cache_control")
|
||||
|
||||
return [strip_litellm_response_only_fields(message) for message in messages]
|
||||
return [
|
||||
strip_litellm_response_only_fields(cast(AllMessageValues, without_cache_control(message)))
|
||||
for message in messages
|
||||
]
|
||||
|
||||
# Cached index of fireworks_ai/* entries from litellm.model_cost. Building
|
||||
# this index requires a full scan of model_cost (tens of thousands of
|
||||
|
|
|
|||
|
|
@ -140,7 +140,8 @@ class GroqChatConfig(OpenAILikeChatConfig):
|
|||
def _transform_messages(
|
||||
self, messages: list[AllMessageValues], model: str, is_async: bool = False
|
||||
) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]:
|
||||
for idx, message in enumerate(messages):
|
||||
transformed_messages: Final = list(messages)
|
||||
for idx, message in enumerate(transformed_messages):
|
||||
"""
|
||||
1. Don't pass 'null' function_call assistant message to groq - https://github.com/BerriAI/litellm/issues/5839
|
||||
"""
|
||||
|
|
@ -154,12 +155,12 @@ class GroqChatConfig(OpenAILikeChatConfig):
|
|||
for k, v in _message.items():
|
||||
if v is not None:
|
||||
new_message[k] = v
|
||||
messages[idx] = strip_litellm_internal_message_fields(new_message)
|
||||
transformed_messages[idx] = strip_litellm_internal_message_fields(new_message)
|
||||
|
||||
if is_async:
|
||||
return super()._transform_messages(messages=messages, model=model, is_async=True)
|
||||
return super()._transform_messages(messages=transformed_messages, model=model, is_async=True)
|
||||
else:
|
||||
return super()._transform_messages(messages=messages, model=model, is_async=False)
|
||||
return super()._transform_messages(messages=transformed_messages, model=model, is_async=False)
|
||||
|
||||
def transform_messages(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
|
||||
"""
|
||||
|
||||
import copy
|
||||
import json
|
||||
from collections.abc import Coroutine
|
||||
from typing import Final, Literal, cast, overload
|
||||
|
|
@ -88,7 +89,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
|
|||
) -> dict:
|
||||
_tools = non_default_params.pop("tools", None)
|
||||
if _tools is not None:
|
||||
_tools = remove_additional_properties(_tools)
|
||||
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
|
||||
_tools = remove_strict_from_schema(_tools)
|
||||
if isinstance(_tools, list):
|
||||
_tools = self._convert_custom_tools_to_function_tools(_tools)
|
||||
|
|
|
|||
|
|
@ -755,7 +755,8 @@ class MistralConfig(OpenAIGPTConfig):
|
|||
if _name is not None:
|
||||
# Remove name if not a tool message
|
||||
if message["role"] != "tool" or isinstance(_name, str) and len(_name.strip()) == 0:
|
||||
message.pop("name", None)
|
||||
message_without_name: Final = {k: v for k, v in message.items() if k != "name"}
|
||||
return cast(AllMessageValues, message_without_name) # cast-ok: the same message minus name
|
||||
|
||||
return message
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from pydantic import TypeAdapter, ValidationError
|
|||
|
||||
import litellm
|
||||
from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS
|
||||
from litellm.litellm_core_utils.core_helpers import map_finish_reason
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
extract_reasoning_content,
|
||||
handle_invalid_parallel_tool_calls,
|
||||
|
|
@ -64,7 +64,7 @@ from litellm.types.utils import (
|
|||
)
|
||||
from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support
|
||||
|
||||
from ..common_utils import OpenAIError
|
||||
from ..common_utils import OpenAIError, without_cache_control
|
||||
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -128,6 +128,29 @@ def _litellm_params_str(litellm_params: Mapping[str, object] | None, key: str) -
|
|||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def _drops_params(litellm_params: Mapping[str, object]) -> bool:
|
||||
return drops_unsupported_params(litellm_params.get("drop_params"))
|
||||
|
||||
|
||||
def _names_claude(model: str, litellm_params: Mapping[str, object]) -> bool:
|
||||
base_model: Final = litellm_params.get("base_model")
|
||||
return "claude" in model.lower() or (isinstance(base_model, str) and "claude" in base_model.lower())
|
||||
|
||||
|
||||
def _claude_messages_within_cache_control_cap(
|
||||
model: str,
|
||||
messages: Sequence[AllMessageValues],
|
||||
tools: Sequence["ChatCompletionToolParam"] | None,
|
||||
litellm_params: Mapping[str, object],
|
||||
request_params: Mapping[str, object],
|
||||
) -> Sequence[AllMessageValues]:
|
||||
if not _names_claude(model, litellm_params) or not _drops_params(litellm_params):
|
||||
return messages
|
||||
from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
|
||||
|
||||
return list(messages_within_cache_control_cap(messages, tools, request_params.get("cache_control"), request_params))
|
||||
|
||||
|
||||
def _prompt_cache_options(optional_params: Mapping[str, object]) -> object | None:
|
||||
try:
|
||||
extra_body: Final = _OBJECT_MAPPING.validate_python(optional_params.get("extra_body"))
|
||||
|
|
@ -311,52 +334,58 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
return content_copy
|
||||
|
||||
async def _async_handle_pdf_url(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
|
||||
file_id: Final = content_item.get("file_id")
|
||||
content_copy: Final = content_item.copy()
|
||||
file_id: Final = content_copy.get("file_id")
|
||||
if file_id is not None: # check for file id being url done in _handle_pdf_url
|
||||
base64_data: Final = await async_convert_url_to_base64(file_id)
|
||||
content_item["file_data"] = base64_data
|
||||
content_item["filename"] = "my_file.pdf"
|
||||
content_item.pop("file_id")
|
||||
return content_item
|
||||
content_copy["file_data"] = base64_data
|
||||
content_copy["filename"] = "my_file.pdf"
|
||||
content_copy.pop("file_id")
|
||||
return content_copy
|
||||
|
||||
def _common_file_data_check(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
|
||||
file_data: Final = content_item.get("file_data")
|
||||
filename: Final = content_item.get("filename")
|
||||
if file_data is not None and filename is None:
|
||||
content_item["filename"] = "my_file.pdf"
|
||||
return {**content_item, "filename": "my_file.pdf"}
|
||||
return content_item
|
||||
|
||||
@staticmethod
|
||||
def _image_item_for_openai(content_item: ChatCompletionImageObject) -> ChatCompletionImageObject:
|
||||
image_url: Final = content_item["image_url"]
|
||||
if isinstance(image_url, str):
|
||||
return {**content_item, "image_url": {"url": image_url}}
|
||||
if not isinstance(image_url, dict):
|
||||
return content_item
|
||||
new_image_url_obj: Final = cast(
|
||||
ChatCompletionImageUrlObject,
|
||||
{k: v for k, v in image_url.items() if k not in {"format", "video_metadata"}},
|
||||
)
|
||||
return {**content_item, "image_url": new_image_url_obj}
|
||||
|
||||
@staticmethod
|
||||
def _file_item_for_openai(content_item: ChatCompletionFileObject) -> ChatCompletionFileObject:
|
||||
litellm_specific_params: Final = {"format", "detail", "video_metadata"}
|
||||
file_obj: Final = content_item.get("file")
|
||||
if file_obj is None:
|
||||
raise litellm.BadRequestError(
|
||||
message="Content block has type='file' but is missing the required 'file' field",
|
||||
model=None,
|
||||
llm_provider="openai",
|
||||
)
|
||||
new_file_obj: Final = ChatCompletionFileObjectFile(
|
||||
**{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
|
||||
)
|
||||
return {**content_item, "file": new_file_obj}
|
||||
|
||||
def _apply_common_transform_content_item(
|
||||
self,
|
||||
content_item: OpenAIMessageContentListBlock,
|
||||
) -> OpenAIMessageContentListBlock:
|
||||
litellm_specific_params: Final = {"format", "detail", "video_metadata"}
|
||||
if content_item.get("type") == "image_url":
|
||||
content_item = cast(ChatCompletionImageObject, content_item)
|
||||
if isinstance(content_item["image_url"], str):
|
||||
content_item["image_url"] = {
|
||||
"url": content_item["image_url"],
|
||||
}
|
||||
elif isinstance(content_item["image_url"], dict):
|
||||
new_image_url_obj: Final = cast(
|
||||
ChatCompletionImageUrlObject,
|
||||
{k: v for k, v in content_item["image_url"].items() if k not in {"format", "video_metadata"}},
|
||||
)
|
||||
content_item["image_url"] = new_image_url_obj
|
||||
elif content_item.get("type") == "file":
|
||||
content_item = cast(ChatCompletionFileObject, content_item)
|
||||
file_obj: Final = content_item.get("file")
|
||||
if file_obj is None:
|
||||
raise litellm.BadRequestError(
|
||||
message="Content block has type='file' but is missing the required 'file' field",
|
||||
model=None,
|
||||
llm_provider="openai",
|
||||
)
|
||||
new_file_obj: Final = ChatCompletionFileObjectFile(
|
||||
**{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
|
||||
)
|
||||
content_item["file"] = new_file_obj
|
||||
|
||||
return self._image_item_for_openai(cast(ChatCompletionImageObject, content_item))
|
||||
if content_item.get("type") == "file":
|
||||
return self._file_item_for_openai(cast(ChatCompletionFileObject, content_item))
|
||||
return content_item
|
||||
|
||||
def _transform_content_item(
|
||||
|
|
@ -419,31 +448,34 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
hoisted_messages: Final = hoist_images_from_tool_messages(stripped_messages)
|
||||
|
||||
async def _async_transform():
|
||||
for message in hoisted_messages:
|
||||
message_content = message.get("content")
|
||||
message_role = message.get("role")
|
||||
|
||||
if message_role == "user" and message_content and isinstance(message_content, list):
|
||||
message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
|
||||
for i, content_item in enumerate(message_content_types):
|
||||
message_content_types[i] = await self._async_transform_content_item(
|
||||
cast(OpenAIMessageContentListBlock, content_item),
|
||||
)
|
||||
return hoisted_messages
|
||||
return [await self._async_user_message_for_openai(message) for message in hoisted_messages]
|
||||
|
||||
if is_async:
|
||||
return _async_transform()
|
||||
else:
|
||||
for message in hoisted_messages:
|
||||
message_content = message.get("content")
|
||||
message_role = message.get("role")
|
||||
if message_role == "user" and message_content and isinstance(message_content, list):
|
||||
message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
|
||||
for i, content_item in enumerate(message_content):
|
||||
message_content_types[i] = self._transform_content_item(
|
||||
cast(OpenAIMessageContentListBlock, content_item)
|
||||
)
|
||||
return hoisted_messages
|
||||
return [self._user_message_for_openai(message) for message in hoisted_messages]
|
||||
|
||||
@staticmethod
|
||||
def _user_content_parts(message: AllMessageValues) -> Sequence[OpenAIMessageContentListBlock] | None:
|
||||
message_content: Final = message.get("content")
|
||||
if message.get("role") != "user" or not message_content or not isinstance(message_content, list):
|
||||
return None
|
||||
return cast(Sequence[OpenAIMessageContentListBlock], message_content)
|
||||
|
||||
def _user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
|
||||
parts: Final = self._user_content_parts(message)
|
||||
if parts is None:
|
||||
return message
|
||||
return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI
|
||||
AllMessageValues, {**message, "content": [self._transform_content_item(part) for part in parts]}
|
||||
)
|
||||
|
||||
async def _async_user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
|
||||
parts: Final = self._user_content_parts(message)
|
||||
if parts is None:
|
||||
return message
|
||||
return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI
|
||||
AllMessageValues, {**message, "content": [await self._async_transform_content_item(part) for part in parts]}
|
||||
)
|
||||
|
||||
def transform_messages(
|
||||
self,
|
||||
|
|
@ -462,23 +494,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
messages: list[AllMessageValues],
|
||||
tools: list["ChatCompletionToolParam"] | None = None,
|
||||
) -> tuple[list[AllMessageValues], list["ChatCompletionToolParam"] | None]:
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
filter_value_from_dict,
|
||||
)
|
||||
from litellm.types.llms.openai import ChatCompletionToolParam
|
||||
|
||||
for i, message in enumerate(messages):
|
||||
messages[i] = cast(
|
||||
AllMessageValues,
|
||||
filter_value_from_dict(message, "cache_control"),
|
||||
)
|
||||
if tools is not None:
|
||||
for i, tool in enumerate(tools):
|
||||
tools[i] = cast(
|
||||
ChatCompletionToolParam,
|
||||
filter_value_from_dict(tool, "cache_control"),
|
||||
)
|
||||
return messages, tools
|
||||
new_messages: Final = [cast(AllMessageValues, without_cache_control(message)) for message in messages]
|
||||
new_tools: Final = (
|
||||
[cast(ChatCompletionToolParam, without_cache_control(tool)) for tool in tools]
|
||||
if tools is not None
|
||||
else None
|
||||
)
|
||||
return new_messages, new_tools
|
||||
|
||||
def translate_developer_role_to_system_role(
|
||||
self,
|
||||
|
|
@ -527,6 +551,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
tools: Sequence["ChatCompletionToolParam"] | None,
|
||||
prompt_cache_options: object,
|
||||
litellm_params: Mapping[str, object],
|
||||
request_params: Mapping[str, object],
|
||||
) -> tuple[Sequence[AllMessageValues], Sequence["ChatCompletionToolParam"] | None]:
|
||||
"""
|
||||
A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on
|
||||
|
|
@ -543,10 +568,11 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
model, custom_llm_provider, api_base, prompt_cache_options
|
||||
)
|
||||
if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base):
|
||||
return messages, tools
|
||||
drops_breakpoints: Final = not reads_breakpoints and (
|
||||
litellm.drop_params is True or litellm_params.get("drop_params") is True
|
||||
)
|
||||
return (
|
||||
_claude_messages_within_cache_control_cap(model, messages, tools, litellm_params, request_params),
|
||||
tools,
|
||||
)
|
||||
drops_breakpoints: Final = not reads_breakpoints and _drops_params(litellm_params)
|
||||
renamed: Final = (
|
||||
[AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
|
||||
if reads_breakpoints
|
||||
|
|
@ -561,7 +587,13 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
if drops_breakpoints
|
||||
else list(tools)
|
||||
)
|
||||
return self.remove_cache_control_flag_from_messages_and_tools(model=model, messages=renamed, tools=kept_tools)
|
||||
sent_messages, sent_tools = self.remove_cache_control_flag_from_messages_and_tools(
|
||||
model=model, messages=renamed, tools=kept_tools
|
||||
)
|
||||
return (
|
||||
_claude_messages_within_cache_control_cap(model, sent_messages, sent_tools, litellm_params, request_params),
|
||||
sent_tools,
|
||||
)
|
||||
|
||||
def _sanitized_tools_update_for_openai(
|
||||
self,
|
||||
|
|
@ -623,6 +655,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
tools=optional_params.get("tools", []),
|
||||
prompt_cache_options=_prompt_cache_options(optional_params),
|
||||
litellm_params=litellm_params,
|
||||
request_params=optional_params,
|
||||
)
|
||||
if tools is not None and len(tools) > 0:
|
||||
optional_params["tools"] = tools
|
||||
|
|
@ -654,6 +687,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
|
|||
tools=optional_params.get("tools", []),
|
||||
prompt_cache_options=_prompt_cache_options(optional_params),
|
||||
litellm_params=litellm_params,
|
||||
request_params=optional_params,
|
||||
)
|
||||
if tools is not None and len(tools) > 0:
|
||||
optional_params["tools"] = tools
|
||||
|
|
|
|||
|
|
@ -153,12 +153,13 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig):
|
|||
- role: system ==> translate to role 'user'
|
||||
"""
|
||||
_supports_system_messages: Final = supports_system_messages(model, "openai")
|
||||
for i, message in enumerate(messages):
|
||||
transformed_messages: Final = list(messages)
|
||||
for i, message in enumerate(transformed_messages):
|
||||
if message["role"] == "system" and not _supports_system_messages:
|
||||
new_message = ChatCompletionUserMessage(content=message["content"], role="user")
|
||||
messages[i] = new_message # Replace the old message with the new one
|
||||
transformed_messages[i] = new_message # Replace the old message with the new one
|
||||
|
||||
if is_async:
|
||||
return super()._transform_messages(messages, model, is_async=cast(Literal[True], True))
|
||||
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[True], True))
|
||||
else:
|
||||
return super()._transform_messages(messages, model, is_async=cast(Literal[False], False))
|
||||
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[False], False))
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Common helpers / utils across al OpenAI endpoints
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import hashlib
|
||||
import inspect
|
||||
import json
|
||||
|
|
@ -31,7 +32,7 @@ if TYPE_CHECKING:
|
|||
import litellm
|
||||
from litellm.exceptions import error_response_without_credentials
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, iter_items
|
||||
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
|
||||
from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
|
|
@ -43,6 +44,47 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
)
|
||||
|
||||
|
||||
def _is_plain_dict(value: object) -> bool:
|
||||
return isinstance(value, dict)
|
||||
|
||||
|
||||
def _is_list(value: object) -> bool:
|
||||
return isinstance(value, list)
|
||||
|
||||
|
||||
def _contains_cache_control(value: object) -> bool:
|
||||
pending: Final[list[object]] = [value] # mutable-ok: a local worklist, never returned or stored
|
||||
while pending:
|
||||
node = pending.pop()
|
||||
mapping = as_str_mapping(node)
|
||||
if mapping is not None:
|
||||
if "cache_control" in mapping:
|
||||
return True
|
||||
pending.extend(mapping.values())
|
||||
elif _is_list(node):
|
||||
pending.extend(iter_items(node))
|
||||
return False
|
||||
|
||||
|
||||
def without_cache_control(item: object) -> object:
|
||||
"""Return `item` with every `cache_control` key removed, without touching the original.
|
||||
|
||||
`filter_value_from_dict` deletes the key in place and recurses into nested dicts and
|
||||
lists, so the caller's own message, tool or input object has to be copied first: it
|
||||
may reuse the same list on a provider that does support prompt caching. The copy is
|
||||
skipped when there is nothing to strip, which keeps the object identity the
|
||||
pass-through paths rely on and avoids a deep copy on every request.
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import filter_value_from_dict
|
||||
|
||||
if not _is_plain_dict(item) or not _contains_cache_control(item):
|
||||
return item
|
||||
copied: Final = copy.deepcopy(item)
|
||||
if isinstance(copied, dict):
|
||||
filter_value_from_dict(copied, "cache_control")
|
||||
return copied
|
||||
|
||||
|
||||
def _get_client_init_params(cls: type) -> tuple[str, ...]:
|
||||
"""Extract __init__ parameter names (excluding 'self') from a class."""
|
||||
return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self")
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@ from openai._base_client import make_request_options
|
|||
from openai._constants import RAW_RESPONSE_HEADER
|
||||
from openai._legacy_response import LegacyAPIResponse
|
||||
from openai._types import RequestOptions
|
||||
from openai.pagination import SyncCursorPage
|
||||
from openai.types import CreateEmbeddingResponse
|
||||
from openai.types.beta.assistant_deleted import AssistantDeleted
|
||||
from openai.types.chat import ChatCompletion, ChatCompletionChunk
|
||||
|
|
@ -40,6 +41,7 @@ from litellm.types.utils import (
|
|||
LiteLLMBatch,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
openai_batch_status,
|
||||
)
|
||||
from litellm.utils import (
|
||||
CustomStreamWrapper,
|
||||
|
|
@ -2182,6 +2184,18 @@ class OpenAIFilesAPI(BaseLLM):
|
|||
return response
|
||||
|
||||
|
||||
def _with_openai_batch_fields(batches: Sequence[Batch]) -> Sequence[Batch]:
|
||||
return [
|
||||
batch.model_copy(
|
||||
update={
|
||||
"object": batch.object or "batch",
|
||||
"status": openai_batch_status(batch.status),
|
||||
}
|
||||
)
|
||||
for batch in batches
|
||||
]
|
||||
|
||||
|
||||
class OpenAIBatchesAPI(BaseLLM):
|
||||
"""
|
||||
OpenAI methods to support for batches
|
||||
|
|
@ -2372,7 +2386,7 @@ class OpenAIBatchesAPI(BaseLLM):
|
|||
):
|
||||
verbose_logger.debug("listing batches, after= %s, limit= %s", after, limit)
|
||||
response: Final = await openai_client.batches.list(after=after, limit=limit)
|
||||
return response
|
||||
return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
|
||||
|
||||
def list_batches(
|
||||
self,
|
||||
|
|
@ -2384,7 +2398,7 @@ class OpenAIBatchesAPI(BaseLLM):
|
|||
organization: str | None,
|
||||
after: str | None = None,
|
||||
limit: int | None = None,
|
||||
client: OpenAI | None = None,
|
||||
client: OpenAI | AsyncOpenAI | None = None,
|
||||
litellm_params: Mapping[str, object] | None = None,
|
||||
):
|
||||
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
|
||||
|
|
@ -2409,7 +2423,9 @@ class OpenAIBatchesAPI(BaseLLM):
|
|||
)
|
||||
return self.alist_batches(openai_client=openai_client, after=after, limit=limit)
|
||||
response: Final = openai_client.batches.list(after=after, limit=limit)
|
||||
return response
|
||||
if not isinstance(response, SyncCursorPage):
|
||||
return response
|
||||
return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
|
||||
|
||||
|
||||
class OpenAIAssistantsAPI(BaseLLM):
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ from litellm.types.router import GenericLiteLLMParams
|
|||
from litellm.types.utils import LlmProviders
|
||||
from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS
|
||||
|
||||
from ..common_utils import OpenAIError
|
||||
from ..common_utils import OpenAIError, without_cache_control
|
||||
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
|
||||
|
||||
OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16
|
||||
|
|
@ -658,13 +658,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search),
|
||||
tools=mapped_tools,
|
||||
)
|
||||
if marker_safe_input is not validated_input:
|
||||
self._strip_caller_input_as_if_unmarked(
|
||||
model=model,
|
||||
caller_input=validated_input,
|
||||
reads_breakpoints=reads_breakpoints,
|
||||
maps_anthropic_tool_search=maps_anthropic_tool_search,
|
||||
)
|
||||
object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools)
|
||||
sanitized_tools: Final = self._sanitized_tool_schemas_for_openai(
|
||||
model=model, tools=object_schema_tools, litellm_params=litellm_params
|
||||
|
|
@ -679,17 +672,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
return marked_input
|
||||
return input_without_unanswered_anthropic_tool_search_calls(marked_input)
|
||||
|
||||
def _strip_caller_input_as_if_unmarked(
|
||||
self,
|
||||
model: str,
|
||||
caller_input: str | ResponseInputParam,
|
||||
reads_breakpoints: bool,
|
||||
maps_anthropic_tool_search: bool,
|
||||
) -> None:
|
||||
self.remove_cache_control_flag_from_input_and_tools(
|
||||
model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search)
|
||||
)
|
||||
|
||||
def _input_with_wire_safe_prompt_cache_breakpoints(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -800,24 +782,29 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
|
|||
the chat path. Strips Anthropic-only `cache_control` markers from
|
||||
Responses API input content blocks and tools.
|
||||
|
||||
`filter_value_from_dict` mutates each dict in place, so the same
|
||||
objects are returned.
|
||||
`filter_value_from_dict` deletes the key in place and recurses, so each
|
||||
item is copied first: the caller keeps its own input list and may reuse
|
||||
it on a provider that does support prompt caching.
|
||||
"""
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
filter_value_from_dict,
|
||||
new_input: Final = (
|
||||
cast( # cast-ok: the comprehension rebuilds the input item for item
|
||||
ResponseInputParam,
|
||||
[without_cache_control(item) for item in input],
|
||||
)
|
||||
if isinstance(input, list)
|
||||
else input
|
||||
)
|
||||
|
||||
if isinstance(input, list):
|
||||
for item in input:
|
||||
if isinstance(item, dict):
|
||||
filter_value_from_dict(cast(dict, item), "cache_control")
|
||||
new_tools: Final = (
|
||||
cast( # cast-ok: the comprehension rebuilds the tool list item for item
|
||||
Sequence[ALL_RESPONSES_API_TOOL_PARAMS],
|
||||
[without_cache_control(tool) for tool in tools],
|
||||
)
|
||||
if tools is not None
|
||||
else None
|
||||
)
|
||||
|
||||
if tools is not None:
|
||||
for tool in tools:
|
||||
if isinstance(tool, dict):
|
||||
filter_value_from_dict(cast(dict, tool), "cache_control")
|
||||
|
||||
return input, tools
|
||||
return new_input, new_tools
|
||||
|
||||
def _drop_bridge_minted_reasoning_items(self, input: str | ResponseInputParam) -> str | ResponseInputParam:
|
||||
if self.custom_llm_provider not in _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING or not isinstance(input, list):
|
||||
|
|
|
|||
|
|
@ -110,38 +110,23 @@ def separate_cached_messages(
|
|||
messages: list[AllMessageValues],
|
||||
) -> tuple[list[AllMessageValues], list[AllMessageValues]]:
|
||||
"""
|
||||
Returns separated cached and non-cached messages.
|
||||
|
||||
Args:
|
||||
messages: List of messages to be separated.
|
||||
|
||||
Returns:
|
||||
Tuple containing:
|
||||
- cached_messages: List of cached messages.
|
||||
- non_cached_messages: List of non-cached messages.
|
||||
Returns the first continuous block of cached messages, plus every system message once
|
||||
any message is cached, since generateContent rejects a system_instruction beside a
|
||||
cachedContent, and the remaining messages.
|
||||
"""
|
||||
cached_messages: list[AllMessageValues] = []
|
||||
non_cached_messages: list[AllMessageValues] = []
|
||||
filtered_messages: Final = [(idx, message) for idx, message in enumerate(messages) if is_cached_message(message)]
|
||||
if not filtered_messages:
|
||||
return [], messages
|
||||
|
||||
# Extract cached messages and their indices
|
||||
filtered_messages: Final[list[tuple[int, AllMessageValues]]] = []
|
||||
for idx, message in enumerate(messages):
|
||||
if is_cached_message(message=message):
|
||||
filtered_messages.append((idx, message))
|
||||
|
||||
# Validate only one block of continuous cached messages
|
||||
last_continuous_block_idx: Final = get_first_continuous_block_idx(filtered_messages)
|
||||
# Separate messages based on the block of cached messages
|
||||
if filtered_messages and last_continuous_block_idx is not None:
|
||||
first_cached_idx: Final = filtered_messages[0][0]
|
||||
last_cached_idx: Final = filtered_messages[last_continuous_block_idx][0]
|
||||
|
||||
cached_messages = messages[first_cached_idx : last_cached_idx + 1]
|
||||
non_cached_messages = messages[:first_cached_idx] + messages[last_cached_idx + 1 :]
|
||||
else:
|
||||
non_cached_messages = messages
|
||||
|
||||
return cached_messages, non_cached_messages
|
||||
cached_block: Final = range(filtered_messages[0][0], filtered_messages[last_continuous_block_idx][0] + 1)
|
||||
goes_in_cache: Final = tuple(
|
||||
idx in cached_block or message.get("role") == "system" for idx, message in enumerate(messages)
|
||||
)
|
||||
return (
|
||||
[message for message, cached in zip(messages, goes_in_cache) if cached],
|
||||
[message for message, cached in zip(messages, goes_in_cache) if not cached],
|
||||
)
|
||||
|
||||
|
||||
def cached_messages_end_on_supported_turn(cached_messages: Sequence[AllMessageValues]) -> bool:
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ Translation from OpenAI's `/chat/completions` endpoint to IBM WatsonX's `/text/c
|
|||
Docs: https://cloud.ibm.com/apidocs/watsonx-ai#text-chat
|
||||
"""
|
||||
|
||||
import copy
|
||||
from typing import Final
|
||||
|
||||
from litellm import verbose_logger
|
||||
|
|
@ -57,7 +58,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig):
|
|||
_tools = non_default_params.pop("tools", None)
|
||||
if _tools is not None:
|
||||
# remove 'additionalProperties' from tools
|
||||
_tools = remove_additional_properties(_tools)
|
||||
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
|
||||
# remove 'strict' from tools
|
||||
_tools = remove_strict_from_schema(_tools)
|
||||
if _tools is not None:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
from collections.abc import AsyncIterator, Iterator, Mapping
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Final
|
||||
|
|
@ -208,7 +209,8 @@ class XAIChatConfig(OpenAIGPTConfig):
|
|||
tools = [
|
||||
filtered_tool
|
||||
for filtered_tool in (
|
||||
_FILTERED_TOOL.validate_python(filter_value_from_dict(tool, "strict")) for tool in value
|
||||
_FILTERED_TOOL.validate_python(filter_value_from_dict(copy.deepcopy(tool), "strict")) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
|
||||
for tool in value
|
||||
)
|
||||
if filtered_tool is not None
|
||||
]
|
||||
|
|
|
|||
|
|
@ -51387,6 +51387,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 5e-06,
|
||||
"output_cost_per_token_batches": 2.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
@ -51415,6 +51420,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 5e-06,
|
||||
"output_cost_per_token_batches": 2.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
@ -52161,6 +52171,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 1.5e-05,
|
||||
"output_cost_per_token_batches": 7.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
@ -52266,6 +52281,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 1.5e-05,
|
||||
"output_cost_per_token_batches": 7.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
|
|||
|
|
@ -5031,8 +5031,9 @@ class ProxyBaseLLMRequestProcessing:
|
|||
if stream_usage is None:
|
||||
return None
|
||||
service_tier: Final = obj.get("service_tier")
|
||||
served_model: Final = litellm_logging_obj.model if litellm_logging_obj is not None else None
|
||||
cost_val: Final = ProxyBaseLLMRequestProcessing._streamed_usage_cost(
|
||||
ModelResponse(usage=stream_usage),
|
||||
ModelResponse(model=served_model, usage=stream_usage),
|
||||
model_name,
|
||||
service_tier if isinstance(service_tier, str) else None,
|
||||
litellm_logging_obj,
|
||||
|
|
|
|||
|
|
@ -52,7 +52,11 @@ from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
|
|||
from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention
|
||||
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
|
||||
from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
AnthropicModelInfo,
|
||||
anthropic_pass_through_api_base,
|
||||
merge_anthropic_beta_headers,
|
||||
)
|
||||
from litellm.llms.azure.passthrough.transformation import (
|
||||
foreign_azure_deployment,
|
||||
is_azure_body_model_inference_endpoint,
|
||||
|
|
@ -1047,7 +1051,7 @@ async def anthropic_proxy_route(
|
|||
"""
|
||||
[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)
|
||||
"""
|
||||
base_target_url = os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
|
||||
base_target_url = anthropic_pass_through_api_base()
|
||||
encoded_endpoint = httpx.URL(endpoint).path
|
||||
|
||||
# Ensure endpoint starts with '/' for proper URL construction
|
||||
|
|
|
|||
|
|
@ -71,7 +71,11 @@ from litellm.litellm_core_utils.litellm_logging import get_masked_values
|
|||
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
|
||||
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
|
||||
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
|
||||
from litellm.llms.anthropic.common_utils import is_anthropic_messages_url
|
||||
from litellm.llms.anthropic.common_utils import (
|
||||
anthropic_accepts_user_id,
|
||||
anthropic_upstream_user_id_limit,
|
||||
is_anthropic_messages_url,
|
||||
)
|
||||
from litellm.llms.base_llm.managed_resources.utils import (
|
||||
resolve_passthrough_managed_id_provider,
|
||||
)
|
||||
|
|
@ -148,6 +152,8 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
|||
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
|
||||
EndpointType,
|
||||
PassthroughStandardLoggingPayload,
|
||||
WebSocketSessionMeter,
|
||||
WebSocketSessionUsage,
|
||||
)
|
||||
from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, CallTypes, Usage
|
||||
|
||||
|
|
@ -501,7 +507,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
|
|||
url: httpx.URL,
|
||||
headers: dict,
|
||||
requested_query_params: dict | None = None,
|
||||
_parsed_body: dict | None = None,
|
||||
_parsed_body: Mapping[str, object] | None = None,
|
||||
forward_multipart: bool = False,
|
||||
) -> httpx.Response:
|
||||
"""
|
||||
|
|
@ -715,8 +721,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
|
|||
# unenforced. Set after the client merge so a request body cannot supply its own budget.
|
||||
#
|
||||
# Only for the built-in provider routes. `get_model_from_request` returns
|
||||
# None for a user-defined pass-through, deliberately: its body is forwarded
|
||||
# verbatim, so `model` there names an UPSTREAM model rather than a
|
||||
# None for a user-defined pass-through, deliberately: the client's `model`
|
||||
# goes upstream unchanged, so it names an UPSTREAM model rather than a
|
||||
# LiteLLM-managed one. Enforcement is therefore skipped on those routes, and
|
||||
# charging a counter anyway would track spend that nothing can refuse, and
|
||||
# would attribute it to a budget the operator scoped to a LiteLLM model that
|
||||
|
|
@ -1185,6 +1191,36 @@ def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
|
|||
return headers
|
||||
|
||||
|
||||
def _json_object(value: object) -> Mapping[str, object]:
|
||||
return cast("Mapping[str, object]", value) if isinstance(value, Mapping) else MappingProxyType({}) # cast-ok: json
|
||||
|
||||
|
||||
def _anthropic_user_id_to_forward(
|
||||
body: Mapping[str, object],
|
||||
url: str,
|
||||
has_pre_call_guardrails: Callable[[Mapping[str, object]], bool],
|
||||
) -> str | None:
|
||||
user_id: Final = _json_object(body.get("metadata")).get("user_id")
|
||||
if not isinstance(user_id, str):
|
||||
return None
|
||||
max_length: Final = anthropic_upstream_user_id_limit(url)
|
||||
if max_length is None:
|
||||
return None
|
||||
if not anthropic_accepts_user_id(user_id, max_length):
|
||||
verbose_proxy_logger.warning("Not forwarding metadata.user_id because the upstream would refuse it")
|
||||
return None
|
||||
if has_pre_call_guardrails(body):
|
||||
verbose_proxy_logger.debug("Not forwarding metadata.user_id because a pre-call guardrail cannot scan it")
|
||||
return None
|
||||
return user_id
|
||||
|
||||
|
||||
def _with_anthropic_user_id(body: Mapping[str, object] | None, user_id: str | None) -> Mapping[str, object] | None:
|
||||
if body is None or user_id is None:
|
||||
return body
|
||||
return {**body, "metadata": {"user_id": user_id}}
|
||||
|
||||
|
||||
def _eu_jev_client_body(custom_llm_provider: str | None, body: Mapping[str, object]) -> Mapping[str, object] | None:
|
||||
return MappingProxyType(dict(body)) if custom_llm_provider == "eu_jev" else None
|
||||
|
||||
|
|
@ -1421,6 +1457,10 @@ async def pass_through_request(
|
|||
request_method=getattr(request, "method", None),
|
||||
cost_per_request=cost_per_request,
|
||||
)
|
||||
hooked_body: Final = _json_object(_parsed_body)
|
||||
client_user_id: Final = _anthropic_user_id_to_forward(
|
||||
hooked_body, str(url), proxy_logging_obj.request_has_pre_call_guardrails
|
||||
)
|
||||
kwargs = HttpPassThroughEndpointHelpers.init_kwargs_for_pass_through_endpoint( # rebind-ok: pre-existing rebinding on a rename-only line
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
_parsed_body=_parsed_body,
|
||||
|
|
@ -1588,11 +1628,13 @@ async def pass_through_request(
|
|||
else:
|
||||
logging_url = str(url) + "?" + requested_query_params_str
|
||||
|
||||
outgoing_body: Final = cast("Mapping[str, object] | None", _parsed_body) # cast-ok: json
|
||||
upstream_body: Final = _with_anthropic_user_id(outgoing_body, client_user_id)
|
||||
logging_obj.pre_call(
|
||||
input=[{"role": "user", "content": safe_dumps(_parsed_body)}],
|
||||
api_key="",
|
||||
additional_args={
|
||||
"complete_input_dict": _parsed_body,
|
||||
"complete_input_dict": upstream_body,
|
||||
"api_base": str(logging_url),
|
||||
"headers": redact_credential_headers(upstream_headers),
|
||||
},
|
||||
|
|
@ -1632,7 +1674,7 @@ async def pass_through_request(
|
|||
url,
|
||||
params=requested_query_params,
|
||||
headers=upstream_headers,
|
||||
json=_parsed_body,
|
||||
json=upstream_body,
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -1731,7 +1773,7 @@ async def pass_through_request(
|
|||
url=url,
|
||||
headers=upstream_headers,
|
||||
requested_query_params=requested_query_params,
|
||||
_parsed_body=_parsed_body,
|
||||
_parsed_body=upstream_body,
|
||||
forward_multipart=is_multipart,
|
||||
)
|
||||
verbose_proxy_logger.debug("response.headers= %s", response.headers)
|
||||
|
|
@ -2595,6 +2637,13 @@ def _with_trace_context(headers: Mapping[str, str], parent_span: object) -> dict
|
|||
return inject_trace_context(headers, parent_span=parent_span)
|
||||
|
||||
|
||||
def _log_session_usage(logging_obj: LiteLLMLoggingObj, usage: WebSocketSessionUsage) -> None:
|
||||
logging_obj.model = usage.model # rebind-ok: the spend logger reads model and cost off the shared logging object
|
||||
logging_obj.model_call_details["model"] = usage.model # rebind-ok: same shared logging object
|
||||
logging_obj.model_call_details["custom_llm_provider"] = usage.custom_llm_provider # rebind-ok: same
|
||||
logging_obj.model_call_details["response_cost"] = usage.response_cost # rebind-ok: same
|
||||
|
||||
|
||||
async def websocket_passthrough_request(
|
||||
websocket: WebSocket,
|
||||
target: str,
|
||||
|
|
@ -2607,6 +2656,7 @@ async def websocket_passthrough_request(
|
|||
setup_model_rewriter: Callable[[str], str] | None = None,
|
||||
builtin_provider_route: bool = False,
|
||||
model: str | None = None,
|
||||
meter: WebSocketSessionMeter | None = None,
|
||||
):
|
||||
"""
|
||||
WebSocket passthrough request handler.
|
||||
|
|
@ -2622,6 +2672,7 @@ async def websocket_passthrough_request(
|
|||
setup_model_rewriter: Optional rewrite of the setup frame's model before it reaches the upstream
|
||||
builtin_provider_route: Whether a built-in provider route serves the connection, so its URL names the provider
|
||||
model: The model the caller named before connecting, logged on a failure before the setup frame arrives
|
||||
meter: Prices the session from the client text frames delivered upstream, for an upstream that reports no usage
|
||||
"""
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj
|
||||
|
|
@ -2632,6 +2683,7 @@ async def websocket_passthrough_request(
|
|||
# Initialize tracking variables
|
||||
start_time: Final = datetime.now()
|
||||
websocket_messages: Final[list[dict[str, object]]] = []
|
||||
metered_total = 0 # rebind-ok: forward_client_to_upstream adds the measure of each client text frame it delivers
|
||||
litellm_call_id: Final = str(uuid.uuid4())
|
||||
|
||||
verbose_proxy_logger.info("WebSocket passthrough (%s): Starting WebSocket connection to %s", endpoint, target)
|
||||
|
|
@ -2750,6 +2802,7 @@ async def websocket_passthrough_request(
|
|||
|
||||
async def forward_client_to_upstream() -> None:
|
||||
"""Forward messages from client to upstream WebSocket"""
|
||||
nonlocal metered_total
|
||||
try:
|
||||
while True:
|
||||
message = await websocket.receive()
|
||||
|
|
@ -2819,6 +2872,8 @@ async def websocket_passthrough_request(
|
|||
# Not a JSON message or doesn't contain setup data
|
||||
|
||||
await upstream_ws.send(_rewrite_vertex_live_setup_model(text_data, setup_model_rewriter))
|
||||
if meter is not None:
|
||||
metered_total += meter.measure(text_data)
|
||||
elif bytes_data is not None:
|
||||
await upstream_ws.send(bytes_data)
|
||||
except asyncio.CancelledError:
|
||||
|
|
@ -2945,6 +3000,8 @@ async def websocket_passthrough_request(
|
|||
self.url = target_url
|
||||
|
||||
mock_response: Final = MockWebSocketResponse(target)
|
||||
if meter is not None:
|
||||
_log_session_usage(logging_obj, meter.price(metered_total))
|
||||
|
||||
# Use the same success handler as HTTP passthrough endpoints
|
||||
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
|
||||
|
|
|
|||
|
|
@ -3031,14 +3031,21 @@ class ProxyLogging:
|
|||
hook to consult, but it judges the payload the same way, so a proxy configured only with
|
||||
one of those still has something to say about every record.
|
||||
"""
|
||||
if request_metadata.get("_guardrail_pipelines"):
|
||||
return self.request_has_pre_call_guardrails({"metadata": dict(request_metadata)}, event_type)
|
||||
|
||||
def request_has_pre_call_guardrails(
|
||||
self,
|
||||
request_data: Mapping[str, object],
|
||||
event_type: GuardrailEventHooks = GuardrailEventHooks.pre_call,
|
||||
) -> bool:
|
||||
if _policy_state_metadata(request_data).get("_guardrail_pipelines"):
|
||||
return True
|
||||
caps: Final = ProxyLogging.callback_capabilities()
|
||||
if caps.has_content_enforcer:
|
||||
return True
|
||||
probe: Final = {"metadata": dict(request_metadata)}
|
||||
return any(
|
||||
isinstance(callback, CustomGuardrail) and callback.should_run_guardrail(data=probe, event_type=event_type)
|
||||
isinstance(callback, CustomGuardrail)
|
||||
and callback.should_run_guardrail(data=request_data, event_type=event_type)
|
||||
for callback in caps.resolved_callbacks
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import uuid
|
|||
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from itertools import accumulate, chain, groupby, islice
|
||||
from itertools import accumulate, chain, groupby
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
|
|
@ -45,7 +45,15 @@ from litellm.constants import (
|
|||
REDACTED_BY_LITELLM,
|
||||
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER,
|
||||
)
|
||||
from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook
|
||||
from litellm.integrations.anthropic_cache_control_hook import (
|
||||
AnthropicCacheControlHook,
|
||||
block_with_cache_control,
|
||||
cache_control_count,
|
||||
dropped_cache_control,
|
||||
newest_marks_within_cache_control_limit,
|
||||
whole_messages_within_cache_control_limit,
|
||||
with_cache_control,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
|
||||
from litellm.litellm_core_utils.get_supported_openai_params import (
|
||||
get_supported_openai_params,
|
||||
|
|
@ -630,10 +638,6 @@ def _validated_cache_control(value: object) -> ChatCompletionCachedContent | Non
|
|||
return None
|
||||
|
||||
|
||||
def _dropped_cache_control(_: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _without_cache_control_key(item: object) -> object:
|
||||
if not isinstance(item, dict):
|
||||
return item
|
||||
|
|
@ -721,83 +725,6 @@ def _content_blocks_carrying(content: object, cache_control: ChatCompletionCache
|
|||
return _with_cache_control_on_last_block(_OBJECT_LIST_ADAPTER.validate_python(content), cache_control)
|
||||
|
||||
|
||||
_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls")
|
||||
|
||||
|
||||
def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]:
|
||||
lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS)
|
||||
return tuple(
|
||||
chain.from_iterable(_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list))
|
||||
)
|
||||
|
||||
|
||||
def _block_carries_cache_control(block: object) -> bool:
|
||||
return isinstance(block, dict) and _STR_KEY_DICT_ADAPTER.validate_python(block).get("cache_control") is not None
|
||||
|
||||
|
||||
def _cache_control_count(message: object) -> int:
|
||||
if not isinstance(message, dict):
|
||||
return 0
|
||||
fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message)
|
||||
block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block))
|
||||
return int(fields.get("cache_control") is not None) + block_marks
|
||||
|
||||
|
||||
_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None]
|
||||
|
||||
|
||||
def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]:
|
||||
others: Final = {key: value for key, value in fields.items() if key != "cache_control"}
|
||||
if "cache_control" not in fields:
|
||||
return others
|
||||
cache_control: Final = rewrite(fields["cache_control"])
|
||||
return others if cache_control is None else {**others, "cache_control": cache_control}
|
||||
|
||||
|
||||
def _block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object:
|
||||
if not isinstance(block, dict):
|
||||
return block
|
||||
return _fields_with_cache_control(_STR_KEY_DICT_ADAPTER.validate_python(block), rewrite)
|
||||
|
||||
|
||||
def _with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT:
|
||||
if not isinstance(message, dict):
|
||||
return message
|
||||
fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message)
|
||||
blocks_rewritten: Final = {
|
||||
key: (
|
||||
[_block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)]
|
||||
if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list)
|
||||
else value
|
||||
)
|
||||
for key, value in fields.items()
|
||||
}
|
||||
return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped
|
||||
_MessageT, _fields_with_cache_control(blocks_rewritten, rewrite)
|
||||
)
|
||||
|
||||
|
||||
def _kept_within_budget(state: tuple[int, bool], marks: int) -> tuple[int, bool]:
|
||||
remaining, _ = state
|
||||
if marks > remaining:
|
||||
return remaining, False
|
||||
return remaining - marks, True
|
||||
|
||||
|
||||
def _messages_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]:
|
||||
budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0)
|
||||
newest_first: Final = tuple(reversed(messages))
|
||||
budget_after_each: Final = accumulate(
|
||||
(_cache_control_count(message) for message in newest_first), _kept_within_budget, initial=(budget, True)
|
||||
)
|
||||
kept_flags: Final = tuple(kept for _, kept in islice(budget_after_each, 1, None))
|
||||
kept_newest_first: Final = tuple(
|
||||
message if kept else _with_cache_control(message, _dropped_cache_control)
|
||||
for message, kept in zip(newest_first, kept_flags)
|
||||
)
|
||||
return tuple(reversed(kept_newest_first))
|
||||
|
||||
|
||||
_SYSTEM_MESSAGE_FIELDS: Final = frozenset(("content", "cache_control"))
|
||||
|
||||
|
||||
|
|
@ -826,7 +753,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT:
|
|||
if fields is None:
|
||||
return message
|
||||
if fields.get("role") == "tool":
|
||||
return _with_cache_control(message, _dropped_cache_control)
|
||||
return with_cache_control(message, dropped_cache_control)
|
||||
tool_calls: Final = fields.get("tool_calls")
|
||||
if not isinstance(tool_calls, list):
|
||||
return message
|
||||
|
|
@ -835,7 +762,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT:
|
|||
{
|
||||
**fields,
|
||||
"tool_calls": [
|
||||
_block_with_cache_control(tool_call, _dropped_cache_control)
|
||||
block_with_cache_control(tool_call, dropped_cache_control)
|
||||
for tool_call in _OBJECT_LIST_ADAPTER.validate_python(tool_calls)
|
||||
],
|
||||
},
|
||||
|
|
@ -849,7 +776,7 @@ class _ItemMarkRules:
|
|||
external_marks: int
|
||||
|
||||
def sanitized(self, message: _MessageT) -> _MessageT:
|
||||
valid: Final = _with_cache_control(message, _validated_cache_control)
|
||||
valid: Final = with_cache_control(message, _validated_cache_control)
|
||||
return _without_tool_block_marks(valid) if self.refuses_tool_block_marks else valid
|
||||
|
||||
def checked(self, message: _MessageT) -> _MessageT:
|
||||
|
|
@ -872,15 +799,15 @@ def _request_messages_for_cache_points(messages: Sequence[_MessageT], rules: _It
|
|||
as_system: Final = tuple(_developer_message_as_system_message(message) for message in messages)
|
||||
if not rules.drop_params:
|
||||
return as_system
|
||||
return _messages_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks)
|
||||
return newest_marks_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks)
|
||||
|
||||
|
||||
def _replayed_messages_for_cache_points(
|
||||
replayed_messages: Sequence[_MessageT], current_messages: Sequence[object], rules: _ItemMarkRules
|
||||
) -> Sequence[_MessageT]:
|
||||
as_system: Final = tuple(_developer_message_as_system_message(message) for message in replayed_messages)
|
||||
current_marks: Final = sum(_cache_control_count(message) for message in current_messages)
|
||||
return _messages_within_cache_control_limit(
|
||||
current_marks: Final = sum(cache_control_count(message) for message in current_messages)
|
||||
return whole_messages_within_cache_control_limit(
|
||||
tuple(map(rules.sanitized, as_system)), current_marks + rules.external_marks
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -403,7 +403,7 @@ class OpenAIFileObject(LiteLLMBaseModel):
|
|||
filename: str
|
||||
"""The name of the file."""
|
||||
|
||||
object: Literal["file"]
|
||||
object: Literal["file"] = "file"
|
||||
"""The object type, which is always `file`."""
|
||||
|
||||
purpose: OpenAIFilesPurpose
|
||||
|
|
@ -441,6 +441,11 @@ class OpenAIFileObject(LiteLLMBaseModel):
|
|||
|
||||
_hidden_params: dict = PrivateAttr(default={"response_cost": 0.0}) # no cost for writing a file
|
||||
|
||||
@field_validator("object", mode="before")
|
||||
@classmethod
|
||||
def _default_missing_object_type(cls, value: _JsonValue) -> _JsonValue:
|
||||
return "file" if value is None else value
|
||||
|
||||
@property
|
||||
def hidden_params(self) -> dict[str, builtins.object]: # mutable-ok: API requires mutation
|
||||
return self._hidden_params
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import Final
|
||||
from typing import Final, Protocol
|
||||
|
||||
from pydantic import Field
|
||||
from typing_extensions import TypedDict
|
||||
|
|
@ -75,6 +76,19 @@ class PassthroughStandardLoggingPayload(TypedDict, total=False):
|
|||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class WebSocketSessionUsage:
|
||||
model: str
|
||||
custom_llm_provider: str
|
||||
response_cost: float | None
|
||||
|
||||
|
||||
class WebSocketSessionMeter(Protocol):
|
||||
def measure(self, client_text: str) -> int: ...
|
||||
|
||||
def price(self, total: int) -> WebSocketSessionUsage: ...
|
||||
|
||||
|
||||
class PassThroughGuardrailSettings(LiteLLMPydanticObjectBase):
|
||||
"""
|
||||
Settings for a specific guardrail on a passthrough endpoint.
|
||||
|
|
|
|||
|
|
@ -106,6 +106,7 @@ from .llms.openai import (
|
|||
OpenAIRealtimeStreamList,
|
||||
ResponsesAPIResponse,
|
||||
WebSearchOptions,
|
||||
_JsonValue,
|
||||
)
|
||||
from .rerank import RerankResponse as RerankResponse
|
||||
|
||||
|
|
@ -4963,29 +4964,50 @@ def _batch_usage_from_chat_usage(usage: Usage) -> BatchUsage:
|
|||
)
|
||||
|
||||
|
||||
OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS: Final = MappingProxyType(
|
||||
{"pending": "validating", "running": "in_progress"}
|
||||
)
|
||||
|
||||
|
||||
def openai_batch_status(status: _JsonValue) -> _JsonValue:
|
||||
return OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(status, status) if isinstance(status, str) else status
|
||||
|
||||
|
||||
class LiteLLMBatch(Batch):
|
||||
_hidden_params: dict = {}
|
||||
|
||||
object: Literal["batch"] = "batch"
|
||||
|
||||
@field_validator("object", mode="before")
|
||||
@classmethod
|
||||
def _default_missing_object_type(cls, value: _JsonValue) -> _JsonValue:
|
||||
return "batch" if value is None else value
|
||||
|
||||
@field_validator("status", mode="before")
|
||||
@classmethod
|
||||
def _map_vllm_router_status(cls, value: _JsonValue) -> _JsonValue:
|
||||
return openai_batch_status(value)
|
||||
|
||||
@property
|
||||
def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation
|
||||
def hidden_params(self) -> dict[str, _JsonValue]: # mutable-ok: API requires mutation
|
||||
return self._hidden_params
|
||||
|
||||
@hidden_params.setter
|
||||
def hidden_params(self, hidden_params: dict[str, object]) -> None: # mutable-ok: API requires mutation
|
||||
def hidden_params(self, hidden_params: dict[str, _JsonValue]) -> None: # mutable-ok: API requires mutation
|
||||
self._hidden_params = hidden_params
|
||||
|
||||
usage: Usage | None = None
|
||||
|
||||
@field_validator("usage", mode="before")
|
||||
@classmethod
|
||||
def _usage_from_batch_token_counts(cls, value: object) -> object:
|
||||
def _usage_from_batch_token_counts(cls, value: _JsonValue) -> _JsonValue:
|
||||
counts: Final = _parse_batch_token_counts(value)
|
||||
return value if counts is None else _usage_with_batch_token_counts(Usage.model_validate(value), counts)
|
||||
|
||||
@field_serializer("usage")
|
||||
def _serialize_usage_with_batch_fields(
|
||||
self, usage: Usage | None, info: FieldSerializationInfo
|
||||
) -> Mapping[str, object] | None:
|
||||
) -> Mapping[str, _JsonValue] | None:
|
||||
if not isinstance(usage, Usage):
|
||||
return usage
|
||||
chat_fields: Final = usage.model_dump(
|
||||
|
|
|
|||
|
|
@ -51387,6 +51387,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 5e-06,
|
||||
"output_cost_per_token_batches": 2.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
@ -51415,6 +51420,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 5e-06,
|
||||
"output_cost_per_token_batches": 2.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
@ -52161,6 +52171,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 1.5e-05,
|
||||
"output_cost_per_token_batches": 7.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
@ -52266,6 +52281,11 @@
|
|||
"max_input_tokens": 200000,
|
||||
"max_output_tokens": 64000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"output_cost_per_token": 1.5e-05,
|
||||
"output_cost_per_token_batches": 7.5e-06,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
|
|
|
|||
|
|
@ -383,6 +383,8 @@ exclude-newer = "3 days"
|
|||
[tool.uv.exclude-newer-package]
|
||||
# TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window
|
||||
brotli = "2026-10-22T14:01:00Z"
|
||||
# TODO: remove after 2026-10-31, once pypdf 6.21.4 clears the 3-day window
|
||||
pypdf = "2026-10-28T17:01:00Z"
|
||||
|
||||
[tool.uv.sources]
|
||||
litellm-proxy-extras = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
PARITY pairs the streamed case with its plain twin where both bill the same row."""
|
||||
|
||||
import json
|
||||
from typing import Final, cast
|
||||
from typing import Final, Literal, cast
|
||||
|
||||
from pydantic import JsonValue
|
||||
|
||||
|
|
@ -41,6 +41,7 @@ from integration.cost_calculation.cases.anthropic import (
|
|||
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
|
||||
from integration.cost_calculation.cost_tracking_case import (
|
||||
CostTrackingTestCase,
|
||||
Deployment,
|
||||
ExactExpected,
|
||||
JsonResponse,
|
||||
RecountExpected,
|
||||
|
|
@ -3387,6 +3388,232 @@ VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS: Final = bridge_pairs(
|
|||
)
|
||||
|
||||
|
||||
VERTEX_AI_CLAUDE_DEPLOYMENT_RATES: Final = Deployment(
|
||||
input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.01
|
||||
)
|
||||
|
||||
|
||||
def _vertex_claude_web_search_message(web_search_requests: int) -> JsonResponse:
|
||||
return JsonResponse(
|
||||
content_type="application/json",
|
||||
body={
|
||||
"id": "msg_$REQUEST_ID",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5-5",
|
||||
"content": [{"type": "text", "text": "scripted answer for cost fixtures"}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {
|
||||
"input_tokens": 1840,
|
||||
"output_tokens": 412,
|
||||
"server_tool_use": {"web_search_requests": web_search_requests},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _vertex_claude_web_search_stream(web_search_requests: int) -> SseResponse:
|
||||
return SseResponse(
|
||||
content_type="text/event-stream",
|
||||
frames=sse_frames(
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": "msg_$REQUEST_ID",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-5-5",
|
||||
"content": [],
|
||||
"stop_reason": None,
|
||||
"usage": {"input_tokens": 1840},
|
||||
},
|
||||
},
|
||||
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": 0,
|
||||
"delta": {"type": "text_delta", "text": "scripted answer for cost fixtures"},
|
||||
},
|
||||
{"type": "content_block_stop", "index": 0},
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": "end_turn"},
|
||||
"usage": {"output_tokens": 412, "server_tool_use": {"web_search_requests": web_search_requests}},
|
||||
},
|
||||
{"type": "message_stop"},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _vertex_claude_deployment_web_search_case(
|
||||
name: str,
|
||||
endpoint: Literal["/v1/chat/completions", "/v1/responses", "/v1/messages"],
|
||||
request: dict[str, JsonValue],
|
||||
response: JsonResponse | SseResponse,
|
||||
tool_usage_cost: float,
|
||||
deployment: Deployment = VERTEX_AI_CLAUDE_DEPLOYMENT_RATES,
|
||||
) -> CostTrackingTestCase:
|
||||
return CostTrackingTestCase(
|
||||
name=name,
|
||||
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
|
||||
model="vertex_ai/claude-sonnet-5-5",
|
||||
endpoint=endpoint,
|
||||
deployment=deployment,
|
||||
request={"model": "$MODEL", **request},
|
||||
response=response,
|
||||
expected=ExactExpected(
|
||||
spend=round(0.0039 + tool_usage_cost, 10),
|
||||
input_cost=0.00184,
|
||||
output_cost=0.00206,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
tool_usage_cost=tool_usage_cost,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL: Final[dict[str, JsonValue]] = {
|
||||
"type": "web_search_20250305",
|
||||
"name": "web_search",
|
||||
"max_uses": 3,
|
||||
}
|
||||
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_rates_messages_web_search_two_requests",
|
||||
"/v1/messages",
|
||||
{
|
||||
"max_tokens": 412,
|
||||
"messages": [{"role": "user", "content": "vertex claude deployment rates messages web search"}],
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
},
|
||||
_vertex_claude_web_search_message(2),
|
||||
0.02,
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM: Final = _vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_rates_messages_web_search_two_requests_stream",
|
||||
"/v1/messages",
|
||||
{
|
||||
"max_tokens": 412,
|
||||
"messages": [{"role": "user", "content": "vertex claude deployment rates messages web search stream"}],
|
||||
"stream": True,
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
},
|
||||
_vertex_claude_web_search_stream(2),
|
||||
0.02,
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE: Final = (
|
||||
_vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_zero_search_rate_messages_two_requests",
|
||||
"/v1/messages",
|
||||
{
|
||||
"max_tokens": 412,
|
||||
"messages": [{"role": "user", "content": "vertex claude deployment zero search rate messages"}],
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
},
|
||||
_vertex_claude_web_search_message(2),
|
||||
0.0,
|
||||
Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.0),
|
||||
)
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY: Final = (
|
||||
_vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_token_rates_only_messages_web_search_two_requests",
|
||||
"/v1/messages",
|
||||
{
|
||||
"max_tokens": 412,
|
||||
"messages": [{"role": "user", "content": "vertex claude deployment token rates only messages web search"}],
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
},
|
||||
_vertex_claude_web_search_message(2),
|
||||
0.034,
|
||||
Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06),
|
||||
)
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_three_requests",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"messages": [{"role": "user", "content": "vertex claude deployment rates chat web search"}],
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
"allowed_openai_params": ["web_search_options"],
|
||||
},
|
||||
_vertex_claude_web_search_message(3),
|
||||
0.03,
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM: Final = _vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_two_requests_stream",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"messages": [{"role": "user", "content": "vertex claude deployment rates chat web search stream"}],
|
||||
"stream": True,
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
"allowed_openai_params": ["web_search_options"],
|
||||
},
|
||||
_vertex_claude_web_search_stream(2),
|
||||
0.02,
|
||||
)
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case(
|
||||
"vertex_ai/claude-sonnet-5-5-deployment_rates_responses_web_search_two_requests",
|
||||
"/v1/responses",
|
||||
{"input": "vertex claude deployment rates responses web search", "tools": [{"type": "web_search"}]},
|
||||
_vertex_claude_web_search_message(2),
|
||||
0.02,
|
||||
)
|
||||
|
||||
GEMINI_3_1_PRO_PREVIEW_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM: Final = CostTrackingTestCase(
|
||||
name="vertex_ai/gemini-3.1-pro-preview-deployment_rates_messages_web_search_two_queries_stream",
|
||||
covers="quota_management.spend_tracking.scripted_wire.logs_cost",
|
||||
model="vertex_ai/gemini-3.1-pro-preview",
|
||||
endpoint="/v1/messages",
|
||||
deployment=Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.03),
|
||||
request={
|
||||
"model": "$MODEL",
|
||||
"max_tokens": 412,
|
||||
"messages": [{"role": "user", "content": "vertex gemini deployment rates messages web search stream"}],
|
||||
"stream": True,
|
||||
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
|
||||
},
|
||||
response=SseResponse(
|
||||
content_type="text/event-stream",
|
||||
frames=sse_frames(
|
||||
{
|
||||
"candidates": [
|
||||
{
|
||||
"content": {"parts": [{"text": "scripted answer for cost fixtures"}], "role": "model"},
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
"modelVersion": "gemini-3.1-pro-preview",
|
||||
},
|
||||
{
|
||||
"candidates": [
|
||||
{
|
||||
"content": {"parts": [{"text": ""}], "role": "model"},
|
||||
"finishReason": "STOP",
|
||||
"index": 0,
|
||||
"groundingMetadata": {"webSearchQueries": ["query 0", "query 1"]},
|
||||
}
|
||||
],
|
||||
"usageMetadata": {
|
||||
"promptTokenCount": 1840,
|
||||
"candidatesTokenCount": 412,
|
||||
"totalTokenCount": 2252,
|
||||
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}],
|
||||
},
|
||||
"modelVersion": "gemini-3.1-pro-preview",
|
||||
},
|
||||
),
|
||||
),
|
||||
expected=ExactExpected(
|
||||
spend=0.0339,
|
||||
input_cost=0.00184,
|
||||
output_cost=0.00206,
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
tool_usage_cost=0.03,
|
||||
),
|
||||
)
|
||||
|
||||
GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT: Final = _gemini_case(
|
||||
GEMINI_3_8_FLASH_INPUT_TEXT, "vertex_ai/gemini-3.1-pro-preview-input_text", "vertex_ai/gemini-3.1-pro-preview",
|
||||
ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412),
|
||||
|
|
@ -3725,6 +3952,14 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
|
|||
VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_STREAM,
|
||||
*(pair.bridged for pair in VERTEX_AI_CLAUDE_SONNET_5_5_BRIDGE_PAIRS),
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM,
|
||||
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH,
|
||||
GEMINI_3_1_PRO_PREVIEW_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM,
|
||||
GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT,
|
||||
GEMINI_3_1_PRO_PREVIEW_STREAM,
|
||||
GEMINI_3_1_PRO_PREVIEW_CACHE_READ,
|
||||
|
|
|
|||
|
|
@ -317,7 +317,6 @@ def test_credentialless_proxy_forwards_the_caller_anthropic_credential_and_never
|
|||
|
||||
|
||||
def test_anthropic_sdk_metadata_user_id_reaches_upstream(api_key_proxy: Gateway, wire: Wire) -> None:
|
||||
pytest.skip("BUG: /anthropic/v1/messages drops the native metadata.user_id field before forwarding upstream")
|
||||
with api_key_proxy.scenario() as scenario:
|
||||
key: Final = scenario.key()
|
||||
sent: Final[list[httpx.Request]] = []
|
||||
|
|
|
|||
|
|
@ -37,13 +37,15 @@ import litellm.batches.main as bm
|
|||
import asyncio
|
||||
import datetime
|
||||
import json
|
||||
from collections.abc import Mapping
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from typing import Final
|
||||
import httpx
|
||||
import respx
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from openai.types.batch import Batch
|
||||
from litellm.types.utils import LiteLLMBatch
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
|
@ -1204,3 +1206,183 @@ async def test_delete_batch_output_file(respx_mock: respx.MockRouter, monkeypatc
|
|||
assert delete_route.call_count == 1
|
||||
assert delete_response.id == "file-output123"
|
||||
assert delete_response.deleted is True
|
||||
|
||||
|
||||
VLLM_ROUTER_API_BASE: Final = "http://vllm-router.test/v1"
|
||||
VLLM_ROUTER_BATCH: Final = MappingProxyType(
|
||||
{
|
||||
"id": "batch_5f8dac",
|
||||
"status": "pending",
|
||||
"input_file_id": "file-b73f4c",
|
||||
"created_at": 1793239207,
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"completion_window": "24h",
|
||||
"output_file_id": None,
|
||||
"total_requests": None,
|
||||
"completed_requests": 0,
|
||||
"failed_requests": 0,
|
||||
"metadata": {},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
async def _create_on_vllm_router() -> LiteLLMBatch:
|
||||
return await litellm.acreate_batch(
|
||||
completion_window="24h",
|
||||
endpoint="/v1/chat/completions",
|
||||
input_file_id="file-b73f4c",
|
||||
custom_llm_provider="hosted_vllm",
|
||||
api_key="sk-test",
|
||||
api_base=VLLM_ROUTER_API_BASE,
|
||||
)
|
||||
|
||||
|
||||
async def _retrieve_from_vllm_router() -> LiteLLMBatch:
|
||||
return await litellm.aretrieve_batch(
|
||||
batch_id="batch_5f8dac", custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE
|
||||
)
|
||||
|
||||
|
||||
VLLM_ROUTER_BATCH_CALLS: Final = (
|
||||
pytest.param("POST", "/batches", _create_on_vllm_router, id="acreate_batch"),
|
||||
pytest.param("GET", "/batches/batch_5f8dac", _retrieve_from_vllm_router, id="aretrieve_batch"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_BATCH_CALLS)
|
||||
@pytest.mark.parametrize(
|
||||
"router_status, openai_status",
|
||||
(
|
||||
("pending", "validating"),
|
||||
("running", "in_progress"),
|
||||
("completed", "completed"),
|
||||
("failed", "failed"),
|
||||
("cancelled", "cancelled"),
|
||||
),
|
||||
)
|
||||
async def test_hosted_vllm_router_batch_comes_back_as_an_openai_batch(
|
||||
respx_mock: respx.MockRouter,
|
||||
method: str,
|
||||
path: str,
|
||||
call_router: Callable[[], Awaitable[LiteLLMBatch]],
|
||||
router_status: str,
|
||||
openai_status: str,
|
||||
):
|
||||
respx_mock.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond(
|
||||
200, json={**VLLM_ROUTER_BATCH, "status": router_status}
|
||||
)
|
||||
|
||||
batch: Final = await call_router()
|
||||
|
||||
assert (batch.id, batch.object, batch.status) == ("batch_5f8dac", "batch", openai_status)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_BATCH_CALLS)
|
||||
@pytest.mark.parametrize("field, value", (("status", "queued"), ("object", "file")))
|
||||
async def test_hosted_vllm_router_batch_with_an_unknown_value_is_rejected(
|
||||
respx_mock: respx.MockRouter,
|
||||
method: str,
|
||||
path: str,
|
||||
call_router: Callable[[], Awaitable[LiteLLMBatch]],
|
||||
field: str,
|
||||
value: str,
|
||||
):
|
||||
respx_mock.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond(
|
||||
200, json={**VLLM_ROUTER_BATCH, field: value}
|
||||
)
|
||||
|
||||
with pytest.raises(ValidationError) as exc_info:
|
||||
await call_router()
|
||||
|
||||
assert [error["loc"] for error in exc_info.value.errors()] == [(field,)]
|
||||
|
||||
|
||||
async def _list_on_vllm_router() -> Sequence[Batch]:
|
||||
page: Final = await litellm.alist_batches(
|
||||
custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE
|
||||
)
|
||||
return page.data
|
||||
|
||||
|
||||
async def _list_on_vllm_router_sync() -> Sequence[Batch]:
|
||||
return litellm.list_batches(custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE).data
|
||||
|
||||
|
||||
@pytest.mark.parametrize("list_router", (_list_on_vllm_router, _list_on_vllm_router_sync))
|
||||
async def test_hosted_vllm_router_batch_list_comes_back_as_openai_batches(
|
||||
respx_mock: respx.MockRouter, list_router: Callable[[], Awaitable[Sequence[Batch]]]
|
||||
):
|
||||
respx_mock.get(f"{VLLM_ROUTER_API_BASE}/batches").respond(
|
||||
200,
|
||||
json={
|
||||
"object": "list",
|
||||
"data": [
|
||||
{**VLLM_ROUTER_BATCH, "id": "batch_5f8dac", "status": "pending"},
|
||||
{**VLLM_ROUTER_BATCH, "id": "batch_c74802", "status": "running"},
|
||||
],
|
||||
"first_id": "batch_5f8dac",
|
||||
"last_id": "batch_c74802",
|
||||
"has_more": False,
|
||||
},
|
||||
)
|
||||
|
||||
batches: Final = await list_router()
|
||||
|
||||
assert [(batch.id, batch.object, batch.status) for batch in batches] == [
|
||||
("batch_5f8dac", "batch", "validating"),
|
||||
("batch_c74802", "batch", "in_progress"),
|
||||
]
|
||||
|
||||
|
||||
async def test_hosted_vllm_router_batch_list_passes_through_a_status_that_is_not_a_string(
|
||||
respx_mock: respx.MockRouter,
|
||||
):
|
||||
respx_mock.get(f"{VLLM_ROUTER_API_BASE}/batches").respond(
|
||||
200,
|
||||
json={
|
||||
"object": "list",
|
||||
"data": [{**VLLM_ROUTER_BATCH, "status": ["pending"]}],
|
||||
"first_id": "batch_5f8dac",
|
||||
"last_id": "batch_5f8dac",
|
||||
"has_more": False,
|
||||
},
|
||||
)
|
||||
|
||||
batches: Final = await _list_on_vllm_router()
|
||||
|
||||
assert [(batch.id, batch.object, batch.status) for batch in batches] == [("batch_5f8dac", "batch", ["pending"])]
|
||||
|
||||
|
||||
async def test_openai_batch_list_items_come_back_as_openai_sent_them(respx_mock: respx.MockRouter):
|
||||
respx_mock.get("http://openai.test/v1/batches").respond(
|
||||
200,
|
||||
json={
|
||||
"object": "list",
|
||||
"data": [
|
||||
{
|
||||
**_OPENAI_BATCH_JSON,
|
||||
"status": "completed",
|
||||
"usage": {
|
||||
"input_tokens": 10,
|
||||
"input_tokens_details": {"cached_tokens": 4},
|
||||
"output_tokens": 5,
|
||||
"output_tokens_details": {"reasoning_tokens": 2},
|
||||
"total_tokens": 15,
|
||||
},
|
||||
},
|
||||
{**_OPENAI_BATCH_JSON, "id": "batch_def456", "status": "queued"},
|
||||
],
|
||||
"first_id": "batch_abc123",
|
||||
"last_id": "batch_def456",
|
||||
"has_more": False,
|
||||
},
|
||||
)
|
||||
|
||||
page: Final = await litellm.alist_batches(
|
||||
custom_llm_provider="openai", api_key="sk-test", api_base="http://openai.test/v1"
|
||||
)
|
||||
|
||||
first, second = page.data
|
||||
assert first.usage is not None
|
||||
assert (first.usage.input_tokens_details.cached_tokens, first.usage.output_tokens_details.reasoning_tokens) == (4, 2)
|
||||
assert (second.id, second.object, second.status) == ("batch_def456", "batch", "queued")
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from collections.abc import Awaitable, Callable
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
|
@ -124,6 +125,61 @@ async def test_afile_retrieve_rejects_a_provider_file_without_its_size():
|
|||
assert [error["loc"] for error in exc_info.value.errors()] == [("bytes",)]
|
||||
|
||||
|
||||
VLLM_ROUTER_API_BASE: Final = "http://vllm-router.test/v1"
|
||||
VLLM_ROUTER_FILE: Final = MappingProxyType(
|
||||
{"id": "file-b73f4c", "bytes": 156, "created_at": 1793238707, "filename": "batch_requests.jsonl", "purpose": "batch"}
|
||||
)
|
||||
|
||||
|
||||
async def _upload_to_vllm_router() -> OpenAIFileObject:
|
||||
return await litellm.acreate_file(
|
||||
file=("batch_requests.jsonl", b'{"custom_id": "request-1"}'),
|
||||
purpose="batch",
|
||||
custom_llm_provider="hosted_vllm",
|
||||
api_key="sk-test",
|
||||
api_base=VLLM_ROUTER_API_BASE,
|
||||
)
|
||||
|
||||
|
||||
async def _retrieve_from_vllm_router() -> OpenAIFileObject:
|
||||
return await litellm.afile_retrieve(
|
||||
file_id="file-b73f4c", custom_llm_provider="hosted_vllm", api_key="sk-test", api_base=VLLM_ROUTER_API_BASE
|
||||
)
|
||||
|
||||
|
||||
VLLM_ROUTER_FILE_CALLS: Final = (
|
||||
pytest.param("POST", "/files", _upload_to_vllm_router, id="acreate_file"),
|
||||
pytest.param("GET", "/files/file-b73f4c", _retrieve_from_vllm_router, id="afile_retrieve"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_FILE_CALLS)
|
||||
@respx.mock
|
||||
async def test_hosted_vllm_file_without_an_object_type_comes_back_as_a_file(
|
||||
method: str, path: str, call_router: Callable[[], Awaitable[OpenAIFileObject]]
|
||||
):
|
||||
respx.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond(200, json=dict(VLLM_ROUTER_FILE))
|
||||
|
||||
file_object: Final = await call_router()
|
||||
|
||||
assert file_object.model_dump(exclude_none=True) == {**VLLM_ROUTER_FILE, "object": "file"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("method, path, call_router", VLLM_ROUTER_FILE_CALLS)
|
||||
@respx.mock
|
||||
async def test_hosted_vllm_file_with_a_different_object_type_is_rejected(
|
||||
method: str, path: str, call_router: Callable[[], Awaitable[OpenAIFileObject]]
|
||||
):
|
||||
respx.route(method=method, url=f"{VLLM_ROUTER_API_BASE}{path}").respond(
|
||||
200, json={**VLLM_ROUTER_FILE, "object": "batch"}
|
||||
)
|
||||
|
||||
with pytest.raises(ValidationError) as exc_info:
|
||||
await call_router()
|
||||
|
||||
assert [error["loc"] for error in exc_info.value.errors()] == [("object",)]
|
||||
|
||||
|
||||
_FILE_BODY: Final = b'{"prompt": "Hello", "completion": "Hi"}'
|
||||
_FINE_TUNE_FILE_JSON: Final = MappingProxyType(
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
from collections.abc import Callable
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
|
@ -7,7 +8,7 @@ from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
|
|||
StandardBuiltInToolCostTracking,
|
||||
)
|
||||
from litellm.types.llms.openai import FileSearchTool, ResponsesAPIResponse, WebSearchOptions
|
||||
from litellm.types.utils import ModelResponse, StandardBuiltInToolsParams
|
||||
from litellm.types.utils import ModelResponse, ServerToolUse, StandardBuiltInToolsParams, Usage
|
||||
|
||||
|
||||
def test_web_search_cost_low():
|
||||
|
|
@ -1052,6 +1053,175 @@ def test_azure_ai_non_claude_web_search_keeps_generic_fallback() -> None:
|
|||
assert get_cost_for_web_search_request(custom_llm_provider="azure_ai", usage=usage, model_info=model_info) is None
|
||||
|
||||
|
||||
def _register_priced_deployment(
|
||||
model: str, search_cost_per_query: float | None, deployment_id: str | None = None
|
||||
) -> str:
|
||||
from litellm import Router
|
||||
|
||||
token_rates: Final = {"input_cost_per_token": 3e-6, "output_cost_per_token": 1.5e-5}
|
||||
search_rates: Final = (
|
||||
{}
|
||||
if search_cost_per_query is None
|
||||
else {
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_low": search_cost_per_query,
|
||||
"search_context_size_medium": search_cost_per_query,
|
||||
"search_context_size_high": search_cost_per_query,
|
||||
}
|
||||
}
|
||||
)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "priced-deployment",
|
||||
"litellm_params": {"model": model, "api_key": "k", "vertex_project": "p", "vertex_location": "global"},
|
||||
"model_info": {**token_rates, **search_rates, **({} if deployment_id is None else {"id": deployment_id})},
|
||||
}
|
||||
]
|
||||
)
|
||||
return router.model_list[0]["model_info"]["id"]
|
||||
|
||||
|
||||
def _claude_chat_response(
|
||||
web_search_requests: int, model: str = "claude-sonnet-5-5"
|
||||
) -> tuple[object, Usage | None]:
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
total_tokens=2252,
|
||||
server_tool_use=ServerToolUse(web_search_requests=web_search_requests),
|
||||
)
|
||||
return ModelResponse(model=model, usage=usage), usage
|
||||
|
||||
|
||||
def _claude_messages_response(
|
||||
web_search_requests: int, model: str = "claude-sonnet-5-5"
|
||||
) -> tuple[object, Usage | None]:
|
||||
return {
|
||||
"id": "msg_vertex_claude_web_search",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": model,
|
||||
"content": [{"type": "text", "text": "scripted answer"}],
|
||||
"usage": {
|
||||
"input_tokens": 1840,
|
||||
"output_tokens": 412,
|
||||
"server_tool_use": {"web_search_requests": web_search_requests},
|
||||
},
|
||||
}, None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("build_response", [_claude_chat_response, _claude_messages_response])
|
||||
@pytest.mark.parametrize(("web_search_requests", "search_cost_per_query"), [(2, 0.01), (3, 0.01), (2, 0.0)])
|
||||
def test_vertex_claude_deployment_priced_web_search_billed_per_request(
|
||||
local_model_cost_map: None,
|
||||
build_response: Callable[[int], tuple[object, Usage | None]],
|
||||
web_search_requests: int,
|
||||
search_cost_per_query: float,
|
||||
) -> None:
|
||||
deployment_id: Final = _register_priced_deployment("vertex_ai/claude-sonnet-5-5", search_cost_per_query)
|
||||
response_object, usage = build_response(web_search_requests)
|
||||
|
||||
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
|
||||
model=deployment_id,
|
||||
response_object=response_object,
|
||||
usage=usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
standard_built_in_tools_params=None,
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(web_search_requests * search_cost_per_query)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("build_response", [_claude_chat_response, _claude_messages_response])
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_model", "served_model", "custom_llm_provider"),
|
||||
# Google bills Claude web search on Sonnet 4.5 and Haiku 4.5 too:
|
||||
# https://cloud.google.com/model-garden/partner-models/pricing (read 2026-10-30)
|
||||
[
|
||||
("vertex_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "vertex_ai"),
|
||||
("vertex_ai/claude-sonnet-4-5@20250929", "claude-sonnet-4-5-20250929", "vertex_ai"),
|
||||
("vertex_ai/claude-haiku-4-5@20251001", "claude-haiku-4-5-20251001", "vertex_ai"),
|
||||
("anthropic/claude-sonnet-5-5", "claude-sonnet-5-5", "anthropic"),
|
||||
("azure_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "azure_ai"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("deployment_id", [None, "claude-token-priced"], ids=["hashed_id", "claude_in_id"])
|
||||
def test_claude_deployment_with_token_rates_only_bills_served_model_search_price(
|
||||
local_model_cost_map: None,
|
||||
build_response: Callable[[int, str], tuple[object, Usage | None]],
|
||||
deployment_model: str,
|
||||
served_model: str,
|
||||
custom_llm_provider: str,
|
||||
deployment_id: str | None,
|
||||
) -> None:
|
||||
served_search_pricing: Final = (
|
||||
litellm.get_model_info(model=deployment_model).get("search_context_cost_per_query") or {}
|
||||
)
|
||||
served_search_price: Final = served_search_pricing.get("search_context_size_medium") or 0.0
|
||||
registered_id: Final = _register_priced_deployment(deployment_model, None, deployment_id)
|
||||
response_object, usage = build_response(2, served_model)
|
||||
|
||||
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
|
||||
model=registered_id,
|
||||
response_object=response_object,
|
||||
usage=usage,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
standard_built_in_tools_params=None,
|
||||
)
|
||||
|
||||
assert served_search_price > 0
|
||||
assert cost == pytest.approx(2 * served_search_price)
|
||||
|
||||
|
||||
def test_vertex_claude_deployment_with_token_rates_only_and_unmapped_served_model_bills_no_search(
|
||||
local_model_cost_map: None,
|
||||
) -> None:
|
||||
deployment_id: Final = _register_priced_deployment("vertex_ai/claude-sonnet-5-5", None)
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
total_tokens=2252,
|
||||
server_tool_use=ServerToolUse(web_search_requests=2),
|
||||
)
|
||||
|
||||
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
|
||||
model=deployment_id,
|
||||
response_object=ModelResponse(model="claude-unmapped-preview", usage=usage),
|
||||
usage=usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
standard_built_in_tools_params=None,
|
||||
)
|
||||
|
||||
assert cost == 0.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("deployment_id", [None, "claude-code-gemini"], ids=["hashed_id", "claude_in_id"])
|
||||
def test_vertex_gemini_deployment_priced_web_search_stays_per_prompt(
|
||||
local_model_cost_map: None, deployment_id: str | None
|
||||
) -> None:
|
||||
search_cost_per_prompt: Final = 0.03
|
||||
registered_id: Final = _register_priced_deployment(
|
||||
"vertex_ai/gemini-2.5-flash", search_cost_per_prompt, deployment_id
|
||||
)
|
||||
usage: Final = Usage(
|
||||
prompt_tokens=1840,
|
||||
completion_tokens=412,
|
||||
total_tokens=2252,
|
||||
server_tool_use=ServerToolUse(web_search_requests=2),
|
||||
)
|
||||
|
||||
cost: Final = StandardBuiltInToolCostTracking.get_cost_for_built_in_tools(
|
||||
model=registered_id,
|
||||
response_object=ModelResponse(model="gemini-2.5-flash", usage=usage),
|
||||
usage=usage,
|
||||
custom_llm_provider="vertex_ai",
|
||||
standard_built_in_tools_params=None,
|
||||
)
|
||||
|
||||
assert cost == pytest.approx(search_cost_per_prompt)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kwargs",
|
||||
[{"tools": 5}, {"web_search_options": None}, {"web_search_options": 5}],
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
get_semantic_cache_prompt_from_messages,
|
||||
get_str_from_messages,
|
||||
handle_any_messages_to_chat_completion_str_messages_conversion,
|
||||
handle_messages_with_content_list_to_str_conversion,
|
||||
hoist_images_from_tool_messages,
|
||||
is_encrypted_reasoning_block,
|
||||
merge_consecutive_system_messages,
|
||||
|
|
@ -2675,3 +2676,16 @@ def test_tpm_projection_preserves_non_media_identity_and_opaque_values() -> None
|
|||
projected, media = split_media_for_tpm(request)
|
||||
assert projected is request
|
||||
assert media == ()
|
||||
|
||||
|
||||
def test_content_list_flattening_returns_new_messages_and_leaves_the_callers_untouched() -> None:
|
||||
messages: Final = [
|
||||
{"role": "user", "content": [{"type": "text", "text": "Hello"}, {"type": "text", "text": " world"}]},
|
||||
{"role": "assistant", "content": "Hi"},
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
flattened: Final = handle_messages_with_content_list_to_str_conversion(messages)
|
||||
|
||||
assert flattened == [{"role": "user", "content": "Hello world"}, {"role": "assistant", "content": "Hi"}]
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import copy
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Iterator
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
|
@ -9919,3 +9920,186 @@ def test_transform_request_tool_config_point_without_tools_drops_the_key():
|
|||
assert "cache_control_injection_points" not in data
|
||||
assert "tools" not in data
|
||||
assert litellm_params["metadata"] == {}
|
||||
|
||||
|
||||
def _five_cache_hint_conversation() -> list[dict[str, object]]:
|
||||
return [
|
||||
{"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "assistant", "content": "answer 2"},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]},
|
||||
]
|
||||
|
||||
|
||||
def _cached_texts(node: object) -> Iterator[str]:
|
||||
if isinstance(node, dict):
|
||||
yield from ((node["text"],) if "cache_control" in node and "text" in node else ())
|
||||
yield from chain.from_iterable(map(_cached_texts, node.values()))
|
||||
elif isinstance(node, list):
|
||||
yield from chain.from_iterable(map(_cached_texts, node))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("config", "model"),
|
||||
[
|
||||
(AnthropicConfig(), "claude-haiku-5-5"),
|
||||
(VertexAIAnthropicConfig(), "claude-haiku-5-5"),
|
||||
(AmazonAnthropicClaudeConfig(), "anthropic.claude-haiku-5-5"),
|
||||
(AzureAnthropicConfig(), "claude-haiku-5-5"),
|
||||
],
|
||||
ids=["anthropic", "vertex_ai", "bedrock_invoke", "azure_ai"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
("drop_params", "expected_cached_texts"),
|
||||
[
|
||||
(True, ["answer 1", "turn 1", "turn 2", "turn 3"]),
|
||||
("true", ["answer 1", "turn 1", "turn 2", "turn 3"]),
|
||||
(False, ["answer 1", "system prompt", "turn 1", "turn 2", "turn 3"]),
|
||||
],
|
||||
ids=[
|
||||
"drop_params_keeps_the_newest_four",
|
||||
"string_drop_params_keeps_the_newest_four",
|
||||
"without_drop_params_all_five_pass_through",
|
||||
],
|
||||
)
|
||||
def test_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
|
||||
config: AnthropicConfig,
|
||||
model: str,
|
||||
drop_params: bool | str,
|
||||
expected_cached_texts: list[str],
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
messages: Final = _five_cache_hint_conversation()
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = config.transform_request(
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": drop_params},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert sorted(_cached_texts(body)) == expected_cached_texts
|
||||
assert json.dumps(body).count('"cache_control"') == len(expected_cached_texts)
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def test_claude_cache_control_cap_keeps_a_cached_tool_and_drops_the_oldest_message_hint(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
messages: Final = _five_cache_hint_conversation()[1:]
|
||||
tool: Final = {
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather",
|
||||
"input_schema": {"type": "object", "properties": {}},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
|
||||
body: Final = AnthropicConfig().transform_request(
|
||||
model="claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={"tools": [tool]},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert body["tools"][0]["cache_control"] == {"type": "ephemeral"}
|
||||
assert sorted(_cached_texts(body["messages"])) == ["answer 1", "turn 2", "turn 3"]
|
||||
|
||||
|
||||
def test_claude_cache_control_cap_drops_a_string_assistant_turns_text_hint_before_its_tool_call_hints(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
tool_calls: Final = [
|
||||
{"id": f"call_{name}", "type": "function", "function": {"name": name, "arguments": "{}"}, "cache_control": hint}
|
||||
for name in ("lookup", "search")
|
||||
]
|
||||
follow_up: Final = [
|
||||
{"type": "text", "text": "follow-up", "cache_control": hint},
|
||||
{"type": "text", "text": "and more", "cache_control": hint},
|
||||
]
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "question"},
|
||||
{"role": "assistant", "content": "checking both", "cache_control": hint, "tool_calls": tool_calls},
|
||||
{"role": "tool", "tool_call_id": "call_lookup", "content": "found"},
|
||||
{"role": "tool", "tool_call_id": "call_search", "content": "none"},
|
||||
{"role": "user", "content": follow_up},
|
||||
]
|
||||
|
||||
body: Final = AnthropicConfig().transform_request(
|
||||
model="claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assistant_blocks: Final = body["messages"][1]["content"]
|
||||
assert [block["type"] for block in assistant_blocks] == ["text", "tool_use", "tool_use"]
|
||||
assert ["cache_control" in block for block in assistant_blocks] == [False, True, True]
|
||||
assert json.dumps(body).count('"cache_control"') == 4
|
||||
assert json.dumps(body).count('"cache_control"') == 4
|
||||
|
||||
|
||||
def test_claude_cache_control_cap_keeps_the_newest_hints_inside_one_message(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
messages: Final = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
*({"type": "text", "text": f"part {n}", "cache_control": {"type": "ephemeral"}} for n in range(1, 6)),
|
||||
{"type": "text", "text": "Reply with the single word OK."},
|
||||
],
|
||||
}
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = AnthropicConfig().transform_request(
|
||||
model="claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert sorted(_cached_texts(body["messages"])) == ["part 2", "part 3", "part 4", "part 5"]
|
||||
assert json.dumps(body).count('"cache_control"') == 4
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def test_claude_cache_control_cap_does_not_count_a_message_level_hint_that_list_content_never_sends(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
messages: Final = [
|
||||
{"role": "system", "content": "system prompt", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "user", "content": "turn 1", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "assistant", "content": "answer 1"},
|
||||
{"role": "user", "content": "turn 2", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "assistant", "content": "answer 2"},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}],
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
]
|
||||
|
||||
body: Final = AnthropicConfig().transform_request(
|
||||
model="claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": True},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert sorted(_cached_texts(body)) == ["system prompt", "turn 1", "turn 2", "turn 3"]
|
||||
assert json.dumps(body).count('"cache_control"') == 4
|
||||
|
|
|
|||
|
|
@ -56,6 +56,98 @@ def test_is_anthropic_messages_url(url: str, expected: bool) -> None:
|
|||
assert is_anthropic_messages_url(url) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("configured_base", "url", "expected"),
|
||||
[
|
||||
pytest.param(None, "https://api.anthropic.com/v1/messages", 512, id="default-upstream"),
|
||||
pytest.param(None, "https://api.anthropic.com:443/v1/messages/", 512, id="default-upstream-explicit-port"),
|
||||
pytest.param("http://relay.test:8080", "https://api.anthropic.com/v1/messages", 512, id="direct-target"),
|
||||
pytest.param("https://api.anthropic.com", "https://api.anthropic.com/v1/messages", 512, id="configured-default"),
|
||||
pytest.param("http://relay.test:8080", "http://relay.test:8080/v1/messages", 255, id="configured-upstream"),
|
||||
pytest.param("http://relay.test:8080/a", "http://relay.test:8080/a/v1/messages", 255, id="configured-prefix"),
|
||||
pytest.param("https://Relay.Test", "https://relay.test/v1/messages", 255, id="configured-upper-case-host"),
|
||||
pytest.param("https://relay.test:443", "https://relay.test/v1/messages", 255, id="configured-https-port"),
|
||||
pytest.param("http://relay.test:80/a", "http://relay.test/a/v1/messages", 255, id="configured-http-port"),
|
||||
pytest.param("http://relay.test:8080", "http://relay.test:8081/v1/messages", None, id="other-port"),
|
||||
pytest.param("http://relay.test:notaport", "http://relay.test/v1/messages", None, id="invalid-configured-base"),
|
||||
pytest.param(None, "https://other-vendor.test/v1/messages", None, id="other-vendor"),
|
||||
pytest.param(None, "https://api.anthropic.com/v1/messages/count_tokens", None, id="count-tokens"),
|
||||
],
|
||||
)
|
||||
def test_anthropic_upstream_user_id_limit(
|
||||
monkeypatch: pytest.MonkeyPatch, configured_base: str | None, url: str, expected: int | None
|
||||
) -> None:
|
||||
from litellm.llms.anthropic.common_utils import anthropic_upstream_user_id_limit
|
||||
|
||||
monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False)
|
||||
if configured_base is not None:
|
||||
monkeypatch.setenv("ANTHROPIC_API_BASE", configured_base)
|
||||
|
||||
assert anthropic_upstream_user_id_limit(url) == expected
|
||||
|
||||
|
||||
def test_anthropic_pass_through_api_base_prefers_the_api_base_variable(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
from litellm.llms.anthropic.common_utils import anthropic_pass_through_api_base
|
||||
|
||||
monkeypatch.setenv("ANTHROPIC_BASE_URL", "http://fallback.test")
|
||||
monkeypatch.setenv("ANTHROPIC_API_BASE", "http://preferred.test")
|
||||
assert anthropic_pass_through_api_base() == "http://preferred.test"
|
||||
|
||||
monkeypatch.delenv("ANTHROPIC_API_BASE")
|
||||
assert anthropic_pass_through_api_base() == "http://fallback.test"
|
||||
|
||||
|
||||
# Each row was sent to https://api.anthropic.com/v1/messages on 2026-10-30, False rows answered 400.
|
||||
@pytest.mark.parametrize(
|
||||
("user_id", "expected"),
|
||||
[
|
||||
pytest.param("user-1", True, id="opaque"),
|
||||
pytest.param("", True, id="empty"),
|
||||
pytest.param('{"device_id":"abc","session_id":"def"}', True, id="claude-code-shape"),
|
||||
pytest.param("1234567890", True, id="digits"),
|
||||
pytest.param("+14155550100", True, id="phone-shape"),
|
||||
pytest.param("a@b", True, id="no-dot"),
|
||||
pytest.param("@b.co", True, id="nothing-before-at-sign"),
|
||||
pytest.param("a@.co", True, id="nothing-between-at-sign-and-dot"),
|
||||
pytest.param("a@b.", True, id="nothing-after-dot"),
|
||||
pytest.param("a @ b.co", True, id="spaces-around-at-sign"),
|
||||
pytest.param("a\u00a0@b.co", False, id="no-break-space-before-at-sign"),
|
||||
pytest.param("a@b\n.co", False, id="newline-before-dot"),
|
||||
pytest.param("line one\nline two", False, id="newline"),
|
||||
pytest.param("a\tb", False, id="tab"),
|
||||
pytest.param("a\x1f@b.co", False, id="unit-separator-before-at-sign"),
|
||||
pytest.param("a@\x1cb.co", False, id="file-separator-after-at-sign"),
|
||||
pytest.param("a\u200bb", False, id="zero-width-space"),
|
||||
pytest.param("a@b.co", False, id="email"),
|
||||
pytest.param("user a@b.co here", False, id="email-inside-text"),
|
||||
pytest.param('{"device_id":"a@b.co"}', False, id="email-inside-json"),
|
||||
pytest.param("a@@example.com", False, id="doubled-at-sign"),
|
||||
pytest.param("@@example.com", False, id="leading-doubled-at-sign"),
|
||||
pytest.param("a@b@c.co", False, id="two-at-signs"),
|
||||
pytest.param("a@b..co", False, id="doubled-dot"),
|
||||
pytest.param("a@.b.co", False, id="dot-after-at-sign"),
|
||||
pytest.param("a@b.c d", False, id="email-before-space"),
|
||||
],
|
||||
)
|
||||
def test_anthropic_accepts_user_id(user_id: str, expected: bool) -> None:
|
||||
from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH
|
||||
from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id
|
||||
|
||||
assert anthropic_accepts_user_id(user_id, ANTHROPIC_MAX_USER_ID_LENGTH) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("character", ["a", "\u00e9", "\U0001f600"], ids=["ascii", "two-bytes", "four-bytes"])
|
||||
def test_anthropic_accepts_user_id_counts_characters_up_to_the_limit(character: str) -> None:
|
||||
from litellm.constants import ANTHROPIC_MAX_USER_ID_LENGTH
|
||||
from litellm.llms.anthropic.common_utils import anthropic_accepts_user_id
|
||||
|
||||
assert anthropic_accepts_user_id(character * ANTHROPIC_MAX_USER_ID_LENGTH, ANTHROPIC_MAX_USER_ID_LENGTH) is True
|
||||
assert (
|
||||
anthropic_accepts_user_id(character * (ANTHROPIC_MAX_USER_ID_LENGTH + 1), ANTHROPIC_MAX_USER_ID_LENGTH) is False
|
||||
)
|
||||
|
||||
|
||||
def test_anthropic_model_not_found_maps_to_not_found_but_authentication_error_does_not() -> None:
|
||||
with respx.mock(assert_all_called=True) as router:
|
||||
router.post("https://api.anthropic.com/v1/messages").mock(
|
||||
|
|
|
|||
|
|
@ -11259,3 +11259,86 @@ def test_converse_sends_only_the_header_betas_when_header_and_body_both_carry_be
|
|||
"thinking-binding-controls-2026-08-01",
|
||||
"thinking-display-updates-2026-08-18",
|
||||
]
|
||||
|
||||
|
||||
def _five_cache_hint_conversation() -> list[dict[str, object]]:
|
||||
return [
|
||||
{"role": "system", "content": [{"type": "text", "text": "system prompt", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 1", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "assistant", "content": [{"type": "text", "text": "answer 1", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 2", "cache_control": {"type": "ephemeral"}}]},
|
||||
{"role": "assistant", "content": "answer 2"},
|
||||
{"role": "user", "content": [{"type": "text", "text": "turn 3", "cache_control": {"type": "ephemeral"}}]},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("drop_params", "expected_system_cache_points"),
|
||||
[(True, 0), ("true", 0), (False, 1)],
|
||||
ids=[
|
||||
"drop_params_drops_the_oldest_hint",
|
||||
"string_drop_params_drops_the_oldest_hint",
|
||||
"without_drop_params_all_five_pass_through",
|
||||
],
|
||||
)
|
||||
def test_converse_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
|
||||
drop_params: bool | str, expected_system_cache_points: int, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
messages: Final = _five_cache_hint_conversation()
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
request: Final = AmazonConverseConfig().transform_request(
|
||||
model="anthropic.claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": drop_params},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert len(list(_cache_points(request["system"]))) == expected_system_cache_points
|
||||
assert [len(list(_cache_points(message))) for message in request["messages"]] == [1, 1, 1, 0, 1]
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def test_converse_claude_tool_message_with_several_marked_blocks_counts_as_one_cache_point_against_the_cap(
|
||||
monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
messages: Final = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "policy", "cache_control": {"type": "ephemeral"}},
|
||||
{"type": "text", "text": "question", "cache_control": {"type": "ephemeral"}},
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_1",
|
||||
"content": [
|
||||
{"type": "text", "text": "part 1", "cache_control": {"type": "ephemeral"}},
|
||||
{"type": "text", "text": "part 2", "cache_control": {"type": "ephemeral"}},
|
||||
{"type": "text", "text": "part 3", "cache_control": {"type": "ephemeral"}},
|
||||
],
|
||||
},
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
request: Final = AmazonConverseConfig().transform_request(
|
||||
model="anthropic.claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params={
|
||||
"tools": [{"type": "function", "function": {"name": "lookup", "parameters": {"type": "object"}}}]
|
||||
},
|
||||
litellm_params={"drop_params": True},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert len(list(_cache_points(request["messages"]))) == 3
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -2773,3 +2773,66 @@ def test_claude_json_schema_stream_split_across_json_tools_keeps_the_non_stream_
|
|||
choices: Final = _json_schema_stream_choices(respx_mock, call, split_stream)
|
||||
|
||||
assert [choice.finish_reason for choice in choices if choice.finish_reason] == [answer.finish_reason]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "base_model"),
|
||||
[("databricks-claude-haiku-5-5", None), ("support-bot-endpoint", "databricks/databricks-claude-haiku-5-5")],
|
||||
ids=["claude_in_the_endpoint_name", "claude_named_only_by_base_model"],
|
||||
)
|
||||
def test_databricks_claude_request_over_the_cache_control_cap_keeps_the_newest_four_hints_under_drop_params(
|
||||
model: str, base_model: str | None, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
messages: Final = [
|
||||
{"role": "system", "content": "system prompt", "cache_control": hint},
|
||||
{"role": "user", "content": "turn 1", "cache_control": hint},
|
||||
{"role": "assistant", "content": "answer 1", "cache_control": hint},
|
||||
{"role": "user", "content": "turn 2", "cache_control": hint},
|
||||
{"role": "assistant", "content": "answer 2"},
|
||||
{"role": "user", "content": "turn 3", "cache_control": hint},
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = DatabricksConfig().transform_request(
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": True, "custom_llm_provider": "databricks", "base_model": base_model},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "cache_control" not in json.dumps(body["messages"][0])
|
||||
assert json.dumps(body["messages"]).count('"cache_control"') == 4
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"optional_params",
|
||||
[{"cache_control": {"type": "ephemeral"}}, {"extra_body": {"cache_control": {"type": "ephemeral"}}}],
|
||||
ids=["top_level_cache_control", "extra_body_cache_control"],
|
||||
)
|
||||
def test_databricks_claude_cache_control_cap_keeps_room_for_a_top_level_cache_control_under_drop_params(
|
||||
optional_params: dict[str, object], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "turn 1", "cache_control": hint},
|
||||
{"role": "assistant", "content": "answer 1", "cache_control": hint},
|
||||
{"role": "user", "content": "turn 2", "cache_control": hint},
|
||||
{"role": "assistant", "content": "answer 2", "cache_control": hint},
|
||||
{"role": "user", "content": "turn 3"},
|
||||
]
|
||||
|
||||
body: Final = DatabricksConfig().transform_request(
|
||||
model="databricks-claude-haiku-5-5",
|
||||
messages=messages,
|
||||
optional_params=copy.deepcopy(optional_params),
|
||||
litellm_params={"drop_params": True, "custom_llm_provider": "databricks"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "cache_control" not in json.dumps(body["messages"][0])
|
||||
assert json.dumps(body["messages"]).count('"cache_control"') == 3
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -200,3 +202,38 @@ async def test_deepinfra_tool_message_content_transformation_async():
|
|||
print(f"✓ Async test passed: {tool_message['content']}")
|
||||
|
||||
print("\n✅ DeepInfra async tool message transformation test passed!")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
|
||||
async def test_deepinfra_request_stringifies_tool_content_without_touching_the_callers_message(is_async: bool) -> None:
|
||||
from litellm.llms.deepinfra.chat.transformation import DeepInfraConfig
|
||||
|
||||
config: Final = DeepInfraConfig()
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "Calculate 10 + 10"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"tool_calls": [
|
||||
{"id": "call_1", "type": "function", "function": {"name": "calculator", "arguments": "{}"}}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": [{"type": "text", "text": "20"}]},
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
request_kwargs: Final = {
|
||||
"model": "deepseek-ai/DeepSeek-V3.2",
|
||||
"messages": messages,
|
||||
"optional_params": {},
|
||||
"litellm_params": {},
|
||||
"headers": {},
|
||||
}
|
||||
|
||||
body: Final = (
|
||||
await config.async_transform_request(**request_kwargs)
|
||||
if is_async
|
||||
else config.transform_request(**request_kwargs)
|
||||
)
|
||||
|
||||
assert body["messages"][2]["content"] == "20"
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
|
|
@ -537,6 +538,26 @@ def test_tool_search_reply_sent_to_fireworks_drops_litellm_response_fields():
|
|||
}
|
||||
|
||||
|
||||
def test_cache_control_is_stripped_from_the_request_but_kept_on_the_callers_messages():
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
messages: Final = [
|
||||
{"role": "system", "content": [{"type": "text", "text": "a long cached prompt", "cache_control": hint}]},
|
||||
{"role": "user", "content": "hi", "cache_control": hint},
|
||||
]
|
||||
caller_messages_before: Final = json.dumps(messages)
|
||||
|
||||
request: Final = FireworksAIConfig().transform_request(
|
||||
model="accounts/fireworks/models/gpt-oss-120b",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "cache_control" not in json.dumps(request["messages"])
|
||||
assert json.dumps(messages) == caller_messages_before
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Regression tests for legacy / OpenAPI $ref defs in tool parameters.
|
||||
#
|
||||
|
|
@ -2477,3 +2498,43 @@ def test_translate_developer_role_follows_the_deployment_model_info_over_the_cos
|
|||
)
|
||||
|
||||
assert [message["role"] for message in messages] == expected_roles
|
||||
|
||||
|
||||
def test_fireworks_request_rewrites_tools_without_touching_the_callers_tool() -> None:
|
||||
tools: Final = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "create_thing",
|
||||
"strict": True,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"thing": {"$ref": "#/definitions/Thing"}},
|
||||
"definitions": {"Thing": {"type": "object", "properties": {"id": {"type": "string"}}}},
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
tools_before: Final = copy.deepcopy(tools)
|
||||
|
||||
request: Final = FireworksAIConfig().transform_request(
|
||||
model="accounts/fireworks/models/gpt-oss-120b",
|
||||
messages=[{"role": "user", "content": "Make a thing"}],
|
||||
optional_params={"tools": tools},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["tools"] == [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "create_thing",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"thing": {"type": "object", "properties": {"id": {"type": "string"}}}},
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
assert tools == tools_before
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import logging
|
||||
from typing import Final
|
||||
from unittest.mock import patch
|
||||
|
|
@ -589,3 +590,19 @@ async def test_streamed_reasoning_reply_can_be_replayed(is_async: bool, with_too
|
|||
}
|
||||
]
|
||||
assert reply.reasoning_content == "I should check the weather."
|
||||
|
||||
|
||||
def test_groq_request_drops_null_assistant_fields_without_touching_the_callers_messages() -> None:
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "Hi"},
|
||||
{"role": "assistant", "content": "Hello", "function_call": None, "tool_calls": None},
|
||||
{"role": "user", "content": "Again"},
|
||||
]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = GroqChatConfig().transform_request(
|
||||
model="openai/gpt-oss-120b", messages=messages, optional_params={}, litellm_params={}, headers={}
|
||||
)
|
||||
|
||||
assert body["messages"][1] == {"role": "assistant", "content": "Hello"}
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
import copy
|
||||
import json
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -397,3 +399,43 @@ def test_hosted_vllm_custom_tools_use_top_level_input_schema():
|
|||
assert tools[0]["function"]["name"] == "search"
|
||||
assert tools[0]["function"]["description"] == "Search docs"
|
||||
assert tools[0]["function"]["parameters"] == input_schema
|
||||
|
||||
|
||||
def _strict_weather_tool() -> dict[str, object]:
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"strict": True,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_hosted_vllm_tools_are_rewritten_without_touching_the_callers_tool() -> None:
|
||||
tools: Final = [_strict_weather_tool()]
|
||||
tools_before: Final = copy.deepcopy(tools)
|
||||
|
||||
optional_params: Final = HostedVLLMChatConfig().map_openai_params(
|
||||
non_default_params={"tools": tools}, optional_params={}, model="gpt-oss-120b", drop_params=False
|
||||
)
|
||||
|
||||
assert optional_params["tools"] == [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
assert tools == tools_before
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Mapping
|
||||
|
|
@ -1212,7 +1213,10 @@ def test_mistral_sends_only_the_text_items_of_a_native_thinking_chunk_beside_an_
|
|||
"content": [
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": [{"type": "text", "text": "It has whiskers."}, {"type": "reference", "reference_ids": [1]}],
|
||||
"thinking": [
|
||||
{"type": "text", "text": "It has whiskers."},
|
||||
{"type": "reference", "reference_ids": [1]},
|
||||
],
|
||||
},
|
||||
{"type": "text", "text": "A cat."},
|
||||
],
|
||||
|
|
@ -1672,6 +1676,18 @@ def test_mistral_transform_request_drops_an_empty_tool_message_whose_call_id_is_
|
|||
assert [m.get("role") for m in request["messages"]] == ["user", "assistant"]
|
||||
|
||||
|
||||
def test_mistral_request_drops_a_user_name_without_touching_the_callers_message() -> None:
|
||||
messages: Final = [{"role": "user", "content": "Hi"}, {"role": "user", "content": "", "name": "alice"}]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = MistralConfig().transform_request(
|
||||
model="mistral-medium-latest", messages=messages, optional_params={}, litellm_params={}, headers={}
|
||||
)
|
||||
|
||||
assert "name" not in json.dumps(body["messages"])
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def _warnings_naming(caplog: pytest.LogCaptureFixture, param: str) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
r.getMessage() for r in caplog.records if r.levelno == logging.WARNING and f"Dropping {param}=" in r.getMessage()
|
||||
|
|
|
|||
|
|
@ -1824,6 +1824,94 @@ class TestToolSchemaCombinatorFlatteningForOpenAI:
|
|||
assert request["tools"][0] is tool
|
||||
|
||||
|
||||
class TestCacheControlStrippingDoesNotMutateCallerInput:
|
||||
"""
|
||||
Stripping cache_control for a provider that cannot use it must not reach back
|
||||
into the caller's own message and tool objects.
|
||||
|
||||
filter_value_from_dict deletes the key in place and recurses into nested dicts
|
||||
and lists, and remove_cache_control_flag_from_messages_and_tools assigned the
|
||||
result back into the caller's list, so one call to any OpenAI-compatible
|
||||
provider permanently stripped cache_control from a list the caller still holds.
|
||||
Reusing that list on Anthropic or Bedrock afterwards then silently lost prompt
|
||||
caching, with no error and full-price billing.
|
||||
"""
|
||||
|
||||
def setup_method(self):
|
||||
self.config = OpenAIGPTConfig()
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_openai_base_env(self, monkeypatch):
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None, raising=False)
|
||||
|
||||
@staticmethod
|
||||
def _messages():
|
||||
return [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "a long cached system prompt",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hello",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _tools():
|
||||
return [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {"name": "get_weather", "parameters": {"type": "object"}},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
]
|
||||
|
||||
def _transform(self, messages, tools):
|
||||
return self.config.transform_request(
|
||||
model="gpt-4o",
|
||||
messages=messages,
|
||||
optional_params={"tools": tools},
|
||||
litellm_params={"custom_llm_provider": "openai", "api_base": None},
|
||||
headers={},
|
||||
)
|
||||
|
||||
def test_caller_messages_and_tools_keep_cache_control(self):
|
||||
messages, tools = self._messages(), self._tools()
|
||||
messages_before, tools_before = copy.deepcopy(messages), copy.deepcopy(tools)
|
||||
|
||||
request = self._transform(messages, tools)
|
||||
|
||||
assert "cache_control" not in json.dumps(request)
|
||||
assert messages == messages_before
|
||||
assert tools == tools_before
|
||||
|
||||
def test_a_later_anthropic_call_still_sees_cache_control(self):
|
||||
"""The user-visible consequence: the same message list reused on a provider
|
||||
that does support caching must still carry the cache breakpoints."""
|
||||
messages = self._messages()
|
||||
|
||||
self._transform(messages, self._tools())
|
||||
|
||||
anthropic_body = litellm.AnthropicConfig().transform_request(
|
||||
model="claude-3-5-sonnet-20240620",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
assert "cache_control" in json.dumps(anthropic_body)
|
||||
|
||||
|
||||
def _developer_message_after_a_user_turn() -> list[dict[str, str]]:
|
||||
return [
|
||||
{"role": "system", "content": "You are terse."},
|
||||
|
|
@ -1916,3 +2004,53 @@ def test_completion_leaves_the_callers_closing_developer_message_unchanged_on_a_
|
|||
)
|
||||
assert json.loads(chat.calls.last.request.content)["messages"][-1]["role"] == "user"
|
||||
assert messages == sent
|
||||
|
||||
|
||||
def _user_turn_with_litellm_only_media_fields() -> list[dict[str, object]]:
|
||||
return [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Describe these"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png", "format": "image/png"}},
|
||||
{"type": "image_url", "image_url": "https://example.com/dog.png"},
|
||||
{
|
||||
"type": "file",
|
||||
"file": {"file_data": "data:application/pdf;base64,JVBERi0=", "format": "application/pdf"},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"])
|
||||
async def test_openai_request_rewrites_media_parts_without_touching_the_callers_message(
|
||||
is_async: bool, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", None, raising=False)
|
||||
config: Final = OpenAIGPTConfig()
|
||||
messages: Final = _user_turn_with_litellm_only_media_fields()
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
request_kwargs: Final = {
|
||||
"model": "gpt-5.5",
|
||||
"messages": messages,
|
||||
"optional_params": {},
|
||||
"litellm_params": {"custom_llm_provider": "openai", "api_base": None},
|
||||
"headers": {},
|
||||
}
|
||||
|
||||
body: Final = (
|
||||
await config.async_transform_request(**request_kwargs)
|
||||
if is_async
|
||||
else config.transform_request(**request_kwargs)
|
||||
)
|
||||
|
||||
assert body["messages"][0]["content"] == [
|
||||
{"type": "text", "text": "Describe these"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/cat.png"}},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/dog.png"}},
|
||||
{"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBERi0=", "filename": "my_file.pdf"}},
|
||||
]
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Callable, Mapping
|
||||
|
|
@ -541,6 +542,50 @@ class TestOpenAIResponsesAPIConfig:
|
|||
assert "cache_control" not in result["tools"][0]
|
||||
assert result["tools"][0]["name"] == "get_weather"
|
||||
|
||||
def test_transform_does_not_strip_cache_control_from_the_callers_input(self):
|
||||
"""Stripping for OpenAI must not reach back into the caller's own objects.
|
||||
|
||||
`filter_value_from_dict` deletes the key in place and recurses, and
|
||||
`_validate_input_param` passes plain dict items through by reference, so
|
||||
the caller's input list used to lose its cache breakpoints. Reusing that
|
||||
list on Anthropic or Bedrock afterwards then silently lost prompt caching.
|
||||
Same defect as the Chat Completions path.
|
||||
"""
|
||||
input_with_cache_control = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": "a long cached system prompt",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
tools_with_cache_control = [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_weather",
|
||||
"parameters": {"type": "object"},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
]
|
||||
input_before = copy.deepcopy(input_with_cache_control)
|
||||
tools_before = copy.deepcopy(tools_with_cache_control)
|
||||
|
||||
result = self.config.transform_responses_api_request(
|
||||
model=self.model,
|
||||
input=input_with_cache_control,
|
||||
response_api_optional_request_params={"tools": tools_with_cache_control},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert "cache_control" not in json.dumps(result)
|
||||
assert input_with_cache_control == input_before
|
||||
assert tools_with_cache_control == tools_before
|
||||
|
||||
def test_transform_preserves_input_without_cache_control(self):
|
||||
"""Inputs without cache_control must pass through unmodified."""
|
||||
input_clean = [
|
||||
|
|
@ -4607,10 +4652,10 @@ def test_cache_control_still_marks_a_part_whose_own_marker_is_malformed_under_dr
|
|||
}
|
||||
|
||||
|
||||
def _caller_input_after_transform(model: str, marker: object) -> str:
|
||||
def _hinted_caller_input(marker: object) -> list[dict[str, object]]:
|
||||
hint: Final = {"type": "ephemeral"}
|
||||
marked: Final = {} if marker is None else {"prompt_cache_breakpoint": marker}
|
||||
input: Final = [
|
||||
return [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
|
|
@ -4626,6 +4671,10 @@ def _caller_input_after_transform(model: str, marker: object) -> str:
|
|||
"content": [{"type": "output_text", "text": "ok", "cache_control": hint, **marked}],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _caller_input_after_transform(model: str, marker: object) -> str:
|
||||
input: Final = _hinted_caller_input(marker)
|
||||
_CONFIGS[0].transform_responses_api_request(
|
||||
model=model,
|
||||
input=cast("ResponseInputParam", input), # cast-ok: plain dicts shaped like the SDK's input items
|
||||
|
|
@ -4647,6 +4696,15 @@ def test_a_marker_leaves_the_caller_request_that_fallbacks_and_logs_read_as_an_u
|
|||
assert _caller_input_after_transform(model, marker).replace(marker_json, "") == unmarked
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ("gpt-4.1", "gpt-6.1-sol"))
|
||||
@pytest.mark.parametrize("marker", (None, _VALID_MARKERS[0], "yes"), ids=("unmarked", "explicit", "malformed"))
|
||||
def test_the_caller_input_keeps_its_cache_control_hints_for_a_fallback_deployment(
|
||||
monkeypatch: pytest.MonkeyPatch, model: str, marker: object
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
assert _caller_input_after_transform(model, marker) == json.dumps(_hinted_caller_input(marker))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures(httpx_transport.__name__)
|
||||
async def test_aresponses_forwards_previous_response_id_to_openai() -> None:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -198,3 +199,23 @@ def test_translate_developer_role_keeps_developer_messages_untouched() -> None:
|
|||
)
|
||||
== messages
|
||||
)
|
||||
|
||||
|
||||
def test_o_series_request_turns_system_into_user_without_touching_the_callers_messages(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
model: Final = "o-series-test-model-without-system-role"
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
model,
|
||||
{"litellm_provider": "openai", "mode": "chat", "supports_system_messages": False},
|
||||
)
|
||||
messages: Final = [{"role": "system", "content": "Be terse"}, {"role": "user", "content": "Hi"}]
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = OpenAIOSeriesConfig().transform_request(
|
||||
model=model, messages=messages, optional_params={}, litellm_params={}, headers={}
|
||||
)
|
||||
|
||||
assert [message["role"] for message in body["messages"]] == ["user", "user"]
|
||||
assert messages == messages_before
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ from litellm.llms.openai.openai import (
|
|||
AsyncAssistantEventHandler,
|
||||
AsyncCursorPage,
|
||||
MessageData,
|
||||
OpenAIBatchesAPI,
|
||||
OpenAIChatCompletion,
|
||||
OpenAIConfig,
|
||||
OpenAIMessage as Message,
|
||||
|
|
@ -2320,3 +2321,43 @@ def test_openai_config_hoists_a_later_developer_message_only_for_a_model_off_ope
|
|||
model_info=None,
|
||||
)
|
||||
assert [message["role"] for message in messages] == expected_roles
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_batch_list_given_an_async_client_returns_a_page_the_caller_can_await(
|
||||
http_backend: ModuleType,
|
||||
):
|
||||
def respond(request: _Request) -> httpx.Response | SDKResponse:
|
||||
assert str(request.url).startswith("http://vllm-router.test/v1/batches")
|
||||
return http_backend.Response(
|
||||
200,
|
||||
json={
|
||||
"object": "list",
|
||||
"data": [
|
||||
{
|
||||
"id": "batch_5f8dac",
|
||||
"status": "pending",
|
||||
"input_file_id": "file-b73f4c",
|
||||
"created_at": 1793239207,
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"completion_window": "24h",
|
||||
}
|
||||
],
|
||||
"first_id": "batch_5f8dac",
|
||||
"last_id": "batch_5f8dac",
|
||||
"has_more": False,
|
||||
},
|
||||
)
|
||||
|
||||
async with http_backend.AsyncClient(transport=http_backend.MockTransport(respond)) as http_client:
|
||||
page: Final = OpenAIBatchesAPI().list_batches(
|
||||
_is_async=False,
|
||||
api_key="sk-test",
|
||||
api_base="http://vllm-router.test/v1",
|
||||
timeout=5.0,
|
||||
max_retries=0,
|
||||
organization=None,
|
||||
client=AsyncOpenAI(api_key="sk-test", base_url="http://vllm-router.test/v1", http_client=http_client),
|
||||
)
|
||||
|
||||
assert [(batch.id, batch.status) for batch in (await page).data] == [("batch_5f8dac", "pending")]
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
|
||||
import copy
|
||||
import json
|
||||
from collections.abc import Iterator
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
|
|
@ -874,6 +877,58 @@ def test_openrouter_streaming_reasoning_is_preserved() -> None:
|
|||
assert assembled.choices[0].message.reasoning_content == "Consider the evidence. Apply the rule."
|
||||
|
||||
|
||||
def _five_message_level_cache_hints() -> list[dict[str, object]]:
|
||||
return [
|
||||
{"role": "system", "content": "system prompt", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "user", "content": "turn 1", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "assistant", "content": "answer 1", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "user", "content": "turn 2", "cache_control": {"type": "ephemeral"}},
|
||||
{"role": "assistant", "content": "answer 2"},
|
||||
{"role": "user", "content": "turn 3", "cache_control": {"type": "ephemeral"}},
|
||||
]
|
||||
|
||||
|
||||
def _cached_texts(node: object) -> Iterator[str]:
|
||||
if isinstance(node, dict):
|
||||
yield from ((node["text"],) if "cache_control" in node and "text" in node else ())
|
||||
yield from chain.from_iterable(map(_cached_texts, node.values()))
|
||||
elif isinstance(node, list):
|
||||
yield from chain.from_iterable(map(_cached_texts, node))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("drop_params", "expected_cached_texts"),
|
||||
[
|
||||
(True, ["answer 1", "turn 1", "turn 2", "turn 3"]),
|
||||
("true", ["answer 1", "turn 1", "turn 2", "turn 3"]),
|
||||
(False, ["answer 1", "system prompt", "turn 1", "turn 2", "turn 3"]),
|
||||
],
|
||||
ids=[
|
||||
"drop_params_keeps_the_newest_four",
|
||||
"string_drop_params_keeps_the_newest_four",
|
||||
"without_drop_params_all_five_pass_through",
|
||||
],
|
||||
)
|
||||
def test_openrouter_claude_request_over_the_cache_control_cap_drops_the_oldest_hints_only_under_drop_params(
|
||||
drop_params: bool | str, expected_cached_texts: list[str], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
messages: Final = _five_message_level_cache_hints()
|
||||
messages_before: Final = copy.deepcopy(messages)
|
||||
|
||||
body: Final = OpenrouterConfig().transform_request(
|
||||
model="anthropic/claude-haiku-5.5",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={"drop_params": drop_params, "custom_llm_provider": "openrouter"},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert sorted(_cached_texts(body["messages"])) == expected_cached_texts
|
||||
assert json.dumps(body).count('"cache_control"') == len(expected_cached_texts)
|
||||
assert messages == messages_before
|
||||
|
||||
|
||||
def _transform_openrouter_response(body: dict[str, object]) -> litellm.ModelResponse:
|
||||
return OpenrouterConfig().transform_response(
|
||||
model="openrouter/openai/gpt-4o-mini",
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import json
|
||||
from typing import Final, Literal
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -2507,3 +2508,46 @@ def test_check_and_create_cache_clamps_negative_creation_tokens() -> None:
|
|||
|
||||
assert cache_name == "cachedContents/created"
|
||||
assert logging_obj.vertex_ai_cache_creation_input_tokens == 0
|
||||
|
||||
|
||||
def _recording_gemini_client(calls: list[httpx.Request]) -> HTTPHandler:
|
||||
def route(request: httpx.Request) -> httpx.Response:
|
||||
calls.append(request)
|
||||
if request.url.path.endswith("/cachedContents"):
|
||||
return httpx.Response(200, json={} if request.method == "GET" else _created_cache_body(3000))
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"candidates": [{"content": {"role": "model", "parts": [{"text": "OK"}]}, "finishReason": "STOP"}],
|
||||
"usageMetadata": {"promptTokenCount": 3000, "candidatesTokenCount": 1, "totalTokenCount": 3001},
|
||||
},
|
||||
)
|
||||
|
||||
return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(route)))
|
||||
|
||||
|
||||
def test_a_plain_system_prompt_goes_into_the_gemini_cache_instead_of_beside_it(local_model_cost_map) -> None:
|
||||
calls: Final[list[httpx.Request]] = []
|
||||
litellm.completion(
|
||||
model="gemini/gemini-flash-latest",
|
||||
messages=[
|
||||
{"role": "system", "content": "You are terse."},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "plain system prefix " * 3000, "cache_control": {"type": "ephemeral"}},
|
||||
{"type": "text", "text": "Reply with OK only."},
|
||||
],
|
||||
},
|
||||
],
|
||||
api_key="test-key",
|
||||
client=_recording_gemini_client(calls),
|
||||
)
|
||||
|
||||
created: Final = next(
|
||||
json.loads(call.content) for call in calls if call.method == "POST" and call.url.path.endswith("/cachedContents")
|
||||
)
|
||||
generated: Final = next(json.loads(call.content) for call in calls if call.url.path.endswith(":generateContent"))
|
||||
assert created["system_instruction"] == {"parts": [{"text": "You are terse."}]}
|
||||
assert generated["cachedContent"] == "cachedContents/created"
|
||||
assert "system_instruction" not in generated
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import importlib
|
||||
import json
|
||||
from typing import Final, Optional, cast
|
||||
|
|
@ -10,6 +11,7 @@ import pytest
|
|||
import litellm
|
||||
from litellm import completion, embedding
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.llms.watsonx.chat.transformation import IBMWatsonXChatConfig
|
||||
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
|
||||
|
||||
|
||||
|
|
@ -542,3 +544,43 @@ def test_watsonx_deployment_space_id_embedding(monkeypatch, watsonx_embedding_ca
|
|||
|
||||
# nor space_id or project_id is required by wx.ai API when inferencing deployment
|
||||
assert "project_id" not in json_data and "space_id" not in json_data
|
||||
|
||||
|
||||
def _strict_weather_tool() -> dict[str, object]:
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"strict": True,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_watsonx_chat_tools_are_rewritten_without_touching_the_callers_tool() -> None:
|
||||
tools: Final = [_strict_weather_tool()]
|
||||
tools_before: Final = copy.deepcopy(tools)
|
||||
|
||||
optional_params: Final = IBMWatsonXChatConfig().map_openai_params(
|
||||
non_default_params={"tools": tools}, optional_params={}, model="ibm/granite-4-h-small", drop_params=False
|
||||
)
|
||||
|
||||
assert optional_params["tools"] == [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
assert tools == tools_before
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import copy
|
||||
import json
|
||||
import os
|
||||
from typing import Final
|
||||
|
|
@ -581,3 +582,44 @@ def test_xai_grok_4_frequency_penalty_not_supported(model):
|
|||
"""
|
||||
supported_params = XAIChatConfig().get_supported_openai_params(model=model)
|
||||
assert "frequency_penalty" not in supported_params
|
||||
|
||||
|
||||
def _strict_weather_tool() -> dict[str, object]:
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"strict": True,
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_xai_tools_are_rewritten_without_touching_the_callers_tool() -> None:
|
||||
tools: Final = [_strict_weather_tool()]
|
||||
tools_before: Final = copy.deepcopy(tools)
|
||||
|
||||
optional_params: Final = XAIChatConfig().map_openai_params(
|
||||
non_default_params={"tools": tools}, optional_params={}, model="grok-4.3", drop_params=False
|
||||
)
|
||||
|
||||
assert optional_params["tools"] == [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"city": {"type": "string"}},
|
||||
"required": ["city"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
assert tools == tools_before
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ from starlette.datastructures import UploadFile as StarletteUploadFile
|
|||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.caching.dual_cache import DualCache
|
||||
from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
|
||||
from litellm.constants import ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, ANTHROPIC_MAX_USER_ID_LENGTH, DEFAULT_REQUEST_TIMEOUT_SECONDS
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.prometheus import PrometheusLogger
|
||||
|
|
@ -77,6 +77,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
|
|||
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
|
||||
EndpointType,
|
||||
PassthroughStandardLoggingPayload,
|
||||
WebSocketSessionUsage,
|
||||
)
|
||||
from tests._master_key import MASTER_KEY
|
||||
from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
|
||||
|
|
@ -6620,6 +6621,272 @@ async def test_pass_through_body_stays_byte_identical_outside_the_opt_in(
|
|||
assert "x-litellm-call-id" in response.headers
|
||||
|
||||
|
||||
def _echo_upstream_body(upstream_request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200, content=upstream_request.content, headers={"content-type": "application/json"}, request=upstream_request
|
||||
)
|
||||
|
||||
|
||||
async def _relayed_body(response: Response) -> bytes:
|
||||
if isinstance(response, StreamingResponse):
|
||||
return b"".join([chunk async for chunk in response.body_iterator])
|
||||
return bytes(response.body)
|
||||
|
||||
|
||||
async def _relay_through_pass_through(
|
||||
client_body: Mapping[str, object],
|
||||
target: str,
|
||||
guardrails_config: dict[str, object] | None,
|
||||
custom_body: dict[str, object] | None = None,
|
||||
) -> Mapping[str, object]:
|
||||
async def receive() -> Mapping[str, object]:
|
||||
return {"type": "http.request", "body": json.dumps(dict(client_body)).encode()}
|
||||
|
||||
request: Final = Request(
|
||||
{
|
||||
"type": "http",
|
||||
"method": "POST",
|
||||
"path": "/my-claude-route",
|
||||
"query_string": b"",
|
||||
"headers": [(b"content-type", b"application/json")],
|
||||
},
|
||||
receive,
|
||||
)
|
||||
fake_client, cleanup = _inject_fake_passthrough_client(httpx.MockTransport(_echo_upstream_body), timeout=None)
|
||||
try:
|
||||
with ExitStack() as stack:
|
||||
_enter_relay_logging_mocks(stack, {})
|
||||
stack.enter_context(
|
||||
patch("litellm.proxy.proxy_server.proxy_logging_obj", ProxyLogging(user_api_key_cache=DualCache()))
|
||||
)
|
||||
response: Final = await pass_through_request(
|
||||
request=request,
|
||||
target=target,
|
||||
custom_headers={},
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="sk-relay-test"),
|
||||
custom_body=custom_body,
|
||||
guardrails_config=guardrails_config,
|
||||
)
|
||||
relayed: Final = await _relayed_body(response)
|
||||
finally:
|
||||
cleanup()
|
||||
await fake_client.aclose()
|
||||
|
||||
assert response.status_code == 200
|
||||
return MappingProxyType(json.loads(relayed))
|
||||
|
||||
|
||||
_MESSAGES_BODY: Final = MappingProxyType(
|
||||
{"model": "claude-sonnet-4-5", "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]}
|
||||
)
|
||||
_ANTHROPIC_MESSAGES: Final = "https://api.anthropic.com/v1/messages"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_configured_anthropic_base(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False)
|
||||
monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
@pytest.mark.parametrize("stream", [False, True], ids=["non-streaming", "streaming"])
|
||||
@pytest.mark.parametrize(
|
||||
"client_metadata, target, forwarded_metadata",
|
||||
[
|
||||
({"user_id": "user-1"}, _ANTHROPIC_MESSAGES, {"user_id": "user-1"}),
|
||||
({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/", {"user_id": "user-1"}),
|
||||
({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}?beta=true", {"user_id": "user-1"}),
|
||||
(
|
||||
{"user_id": "user-1", "tags": ["a"], "trace_id": "t", "generation_name": "g", "foo": "bar"},
|
||||
_ANTHROPIC_MESSAGES,
|
||||
{"user_id": "user-1"},
|
||||
),
|
||||
({"user_id": "1234567890"}, _ANTHROPIC_MESSAGES, {"user_id": "1234567890"}),
|
||||
({"tags": ["a"], "foo": "bar"}, _ANTHROPIC_MESSAGES, None),
|
||||
({"user_id": "a@example.com"}, _ANTHROPIC_MESSAGES, None),
|
||||
({"user_id": "x" * (ANTHROPIC_MAX_USER_ID_LENGTH + 1)}, _ANTHROPIC_MESSAGES, None),
|
||||
({"user_id": 42}, _ANTHROPIC_MESSAGES, None),
|
||||
({"user_id": "line one\nline two"}, _ANTHROPIC_MESSAGES, None),
|
||||
({"user_id": "a\x1f@example.com"}, _ANTHROPIC_MESSAGES, None),
|
||||
({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/count_tokens", None),
|
||||
({"user_id": "user-1"}, f"{_ANTHROPIC_MESSAGES}/batches", None),
|
||||
({"user_id": "user-1"}, "https://api.anthropic.com/v1/models", None),
|
||||
({"user_id": "user-1"}, "https://other-vendor.test/anthropic/v1/messages", None),
|
||||
({"user_id": "user-1"}, "https://other-vendor.test/v1/responses", None),
|
||||
({"user_id": "user-1"}, "https://other-vendor.test/v1/chat/completions", None),
|
||||
],
|
||||
ids=[
|
||||
"messages",
|
||||
"messages-trailing-slash",
|
||||
"messages-with-query",
|
||||
"user-id-beside-litellm-and-unknown-keys",
|
||||
"digits-user-id",
|
||||
"no-user-id",
|
||||
"email-user-id",
|
||||
"over-long-user-id",
|
||||
"integer-user-id",
|
||||
"newline-user-id",
|
||||
"control-character-before-at-sign",
|
||||
"count-tokens",
|
||||
"batches",
|
||||
"other-anthropic-path",
|
||||
"other-vendor-messages",
|
||||
"other-vendor-responses",
|
||||
"other-vendor-chat-completions",
|
||||
],
|
||||
)
|
||||
async def test_pass_through_forwards_only_an_acceptable_user_id_to_anthropic_messages(
|
||||
stream: bool, client_metadata: object, target: str, forwarded_metadata: Mapping[str, object] | None
|
||||
) -> None:
|
||||
client_body: Final = MappingProxyType(
|
||||
{**_MESSAGES_BODY, "metadata": client_metadata, **({"stream": True} if stream else {})}
|
||||
)
|
||||
body_without_metadata: Final = {k: v for k, v in client_body.items() if k != "metadata"}
|
||||
|
||||
relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None)
|
||||
|
||||
assert relayed == (
|
||||
body_without_metadata if forwarded_metadata is None else {**body_without_metadata, "metadata": forwarded_metadata}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
@pytest.mark.parametrize("base_env", ["ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"])
|
||||
@pytest.mark.parametrize(
|
||||
"target, forwarded",
|
||||
[
|
||||
("http://gateway.test:8080/anthropic/v1/messages", True),
|
||||
("http://gateway.test:9090/v1/messages", False),
|
||||
("http://other-gateway.test:8080/v1/messages", False),
|
||||
],
|
||||
ids=["configured-upstream", "other-port", "other-host"],
|
||||
)
|
||||
async def test_pass_through_treats_the_configured_anthropic_base_as_anthropic(
|
||||
monkeypatch: pytest.MonkeyPatch, base_env: str, target: str, forwarded: bool
|
||||
) -> None:
|
||||
monkeypatch.setenv(base_env, "http://gateway.test:8080/anthropic")
|
||||
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}})
|
||||
|
||||
relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None)
|
||||
|
||||
assert relayed == (client_body if forwarded else _MESSAGES_BODY)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
@pytest.mark.parametrize(
|
||||
"user_id_length, target, forwarded",
|
||||
[
|
||||
(ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH, "http://gateway.test/v1/messages", True),
|
||||
(ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH + 1, "http://gateway.test/v1/messages", False),
|
||||
(ANTHROPIC_MAX_USER_ID_LENGTH, _ANTHROPIC_MESSAGES, True),
|
||||
],
|
||||
ids=["at-the-configured-base-limit", "over-the-configured-base-limit", "direct-anthropic-keeps-its-limit"],
|
||||
)
|
||||
async def test_pass_through_caps_the_user_id_for_a_configured_anthropic_compatible_base(
|
||||
monkeypatch: pytest.MonkeyPatch, user_id_length: int, target: str, forwarded: bool
|
||||
) -> None:
|
||||
monkeypatch.setenv("ANTHROPIC_API_BASE", "http://gateway.test")
|
||||
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "u" * user_id_length}})
|
||||
|
||||
relayed: Final = await _relay_through_pass_through(client_body, target, guardrails_config=None)
|
||||
|
||||
assert relayed == (client_body if forwarded else _MESSAGES_BODY)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
@pytest.mark.parametrize(
|
||||
"default_on, event_hook, guardrails_config, client_fields, stream_scope, forwarded",
|
||||
[
|
||||
(False, GuardrailEventHooks.pre_call, None, {}, None, True),
|
||||
(False, GuardrailEventHooks.pre_call, {"unregistered-guard": {}}, {}, None, True),
|
||||
(True, GuardrailEventHooks.post_call, None, {}, None, True),
|
||||
(True, GuardrailEventHooks.pre_call, None, {}, None, False),
|
||||
(False, GuardrailEventHooks.pre_call, {"pii-guard": {}}, {}, None, False),
|
||||
(False, GuardrailEventHooks.pre_call, None, {"litellm_metadata": {"guardrails": ["pii-guard"]}}, None, False),
|
||||
(False, GuardrailEventHooks.pre_call, {"pii-guard": {}}, {"litellm_metadata": {"guardrails": []}}, None, False),
|
||||
(True, GuardrailEventHooks.pre_call, None, {"stream": True}, "streaming", False),
|
||||
(True, GuardrailEventHooks.pre_call, None, {}, "streaming", True),
|
||||
(True, GuardrailEventHooks.pre_call, None, {"stream": True}, "non_streaming", True),
|
||||
(True, GuardrailEventHooks.pre_call, None, {}, "non_streaming", False),
|
||||
],
|
||||
ids=[
|
||||
"guardrail-not-requested",
|
||||
"guardrail-not-registered",
|
||||
"default-on-post-call-guardrail",
|
||||
"default-on-pre-call-guardrail",
|
||||
"guardrail-on-the-route",
|
||||
"guardrail-in-litellm-metadata",
|
||||
"route-guardrail-beside-an-empty-client-list",
|
||||
"streaming-guardrail-on-a-stream",
|
||||
"streaming-guardrail-off-a-stream",
|
||||
"non-streaming-guardrail-on-a-stream",
|
||||
"non-streaming-guardrail-off-a-stream",
|
||||
],
|
||||
)
|
||||
async def test_pass_through_keeps_the_user_id_from_routes_a_guardrail_scans(
|
||||
default_on: bool,
|
||||
event_hook: GuardrailEventHooks,
|
||||
guardrails_config: dict[str, object] | None,
|
||||
client_fields: Mapping[str, object],
|
||||
stream_scope: str | None,
|
||||
forwarded: bool,
|
||||
) -> None:
|
||||
guardrail: Final = CustomGuardrail(
|
||||
guardrail_name="pii-guard", default_on=default_on, event_hook=event_hook, stream_scope=stream_scope
|
||||
)
|
||||
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}, **client_fields})
|
||||
body_without_client_fields: Final = {k: v for k, v in client_body.items() if k not in ("metadata", "litellm_metadata")}
|
||||
|
||||
with patch.object(litellm, "callbacks", [guardrail]):
|
||||
relayed: Final = await _relay_through_pass_through(
|
||||
client_body, _ANTHROPIC_MESSAGES, guardrails_config=guardrails_config
|
||||
)
|
||||
|
||||
assert relayed == (
|
||||
{**body_without_client_fields, "metadata": {"user_id": "user-1"}} if forwarded else body_without_client_fields
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
async def test_pass_through_warns_without_the_value_when_it_drops_a_refused_user_id(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "person@example.com"}})
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name):
|
||||
await _relay_through_pass_through(client_body, _ANTHROPIC_MESSAGES, guardrails_config=None)
|
||||
|
||||
assert "Not forwarding metadata.user_id" in caplog.text
|
||||
assert "person@example.com" not in caplog.text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
async def test_pass_through_drops_metadata_that_is_not_an_object() -> None:
|
||||
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": "user-1"})
|
||||
|
||||
relayed: Final = await _relay_through_pass_through(client_body, _ANTHROPIC_MESSAGES, guardrails_config=None)
|
||||
|
||||
assert relayed == _MESSAGES_BODY
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("no_configured_anthropic_base")
|
||||
async def test_pass_through_custom_body_replaces_the_client_user_id() -> None:
|
||||
client_body: Final = MappingProxyType({**_MESSAGES_BODY, "metadata": {"user_id": "user-1"}})
|
||||
|
||||
relayed: Final = await _relay_through_pass_through(
|
||||
client_body, _ANTHROPIC_MESSAGES, guardrails_config=None, custom_body=dict(_MESSAGES_BODY)
|
||||
)
|
||||
|
||||
assert relayed == _MESSAGES_BODY
|
||||
|
||||
|
||||
_PARTIAL_RELAY_WARNING_MARKER = "ended before upstream body was fully relayed"
|
||||
|
||||
|
||||
|
|
@ -7736,6 +8003,102 @@ async def test_websocket_passthrough_sends_deepgram_audio_bytes_and_control_text
|
|||
upstream_ws.close.assert_awaited_once()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _LengthMeter:
|
||||
rate: float
|
||||
|
||||
def measure(self, client_text: str) -> int:
|
||||
return len(client_text)
|
||||
|
||||
def price(self, total: int) -> WebSocketSessionUsage:
|
||||
return WebSocketSessionUsage(model="voice-1", custom_llm_provider="tts-provider", response_cost=total * self.rate)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_passthrough_logs_the_meters_price_for_the_client_text_it_delivered():
|
||||
"""Text frames reach the upstream unchanged and are measured, binary frames are not, and the session is logged at
|
||||
the model, provider and cost the meter prices for the total."""
|
||||
frames = ("hello", b"\x00\x01binary", "world!")
|
||||
upstream_ws = RecordingUpstreamWebSocket()
|
||||
websocket = _client_websocket(
|
||||
AsyncMock(
|
||||
side_effect=[
|
||||
*(
|
||||
{"type": "websocket.receive", "bytes": frame}
|
||||
if isinstance(frame, bytes)
|
||||
else {"type": "websocket.receive", "text": frame}
|
||||
for frame in frames
|
||||
),
|
||||
{"type": "websocket.disconnect"},
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
with (
|
||||
_patched_websocket_passthrough_environment(upstream_ws),
|
||||
patch( # test-quality-ok: pass_through_endpoint_logging is a module global read inside websocket_passthrough_request; there is no injection seam
|
||||
"litellm.proxy.pass_through_endpoints.pass_through_endpoints."
|
||||
"pass_through_endpoint_logging.pass_through_async_success_handler",
|
||||
new=AsyncMock(),
|
||||
) as success_handler,
|
||||
):
|
||||
await websocket_passthrough_request(
|
||||
websocket=websocket,
|
||||
target="wss://upstream.example/v1/stream",
|
||||
custom_headers={},
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
forward_headers=False,
|
||||
endpoint="/provider/v1/stream",
|
||||
accept_websocket=False,
|
||||
meter=_LengthMeter(rate=0.5),
|
||||
)
|
||||
|
||||
assert [call.args[0] for call in upstream_ws.send.await_args_list] == list(frames)
|
||||
logging_obj = success_handler.call_args.kwargs["logging_obj"]
|
||||
assert logging_obj.model == "voice-1"
|
||||
assert logging_obj.model_call_details["model"] == "voice-1"
|
||||
assert logging_obj.model_call_details["custom_llm_provider"] == "tts-provider"
|
||||
assert logging_obj.model_call_details["response_cost"] == (len("hello") + len("world!")) * 0.5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_passthrough_meters_only_client_text_the_upstream_accepted():
|
||||
upstream_ws = RecordingUpstreamWebSocket()
|
||||
upstream_ws.send = AsyncMock(side_effect=[None, ConnectionError("upstream went away")])
|
||||
websocket = _client_websocket(
|
||||
AsyncMock(
|
||||
side_effect=[
|
||||
{"type": "websocket.receive", "text": "hello"},
|
||||
{"type": "websocket.receive", "text": "never delivered"},
|
||||
{"type": "websocket.disconnect"},
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
with (
|
||||
_patched_websocket_passthrough_environment(upstream_ws),
|
||||
patch( # test-quality-ok: pass_through_endpoint_logging is a module global read inside websocket_passthrough_request; there is no injection seam
|
||||
"litellm.proxy.pass_through_endpoints.pass_through_endpoints."
|
||||
"pass_through_endpoint_logging.pass_through_async_success_handler",
|
||||
new=AsyncMock(),
|
||||
) as success_handler,
|
||||
):
|
||||
await websocket_passthrough_request(
|
||||
websocket=websocket,
|
||||
target="wss://upstream.example/v1/stream",
|
||||
custom_headers={},
|
||||
user_api_key_dict=UserAPIKeyAuth(),
|
||||
forward_headers=False,
|
||||
endpoint="/provider/v1/stream",
|
||||
accept_websocket=False,
|
||||
meter=_LengthMeter(rate=0.5),
|
||||
)
|
||||
|
||||
logging_obj = success_handler.call_args.kwargs["logging_obj"]
|
||||
assert [call.args[0] for call in upstream_ws.send.await_args_list] == ["hello", "never delivered"]
|
||||
assert logging_obj.model_call_details["response_cost"] == len("hello") * 0.5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_websocket_passthrough_vertex_live_setup_ack_names_the_model_but_is_not_billed_as_usage():
|
||||
"""Vertex Live keeps its special first frame: the setup acknowledgement is forwarded verbatim, read for the
|
||||
|
|
|
|||
|
|
@ -41,12 +41,13 @@ def _make_streaming_response(chunks):
|
|||
return mock
|
||||
|
||||
|
||||
def _unarmed_logging_obj():
|
||||
def _unarmed_logging_obj(model: str = "unknown"):
|
||||
"""Real Logging objects only carry _on_deferred_stream_complete when the
|
||||
proxy arms deferred dispatch; a bare MagicMock's auto-attribute is truthy
|
||||
and would spuriously trigger the deferral branch."""
|
||||
obj = MagicMock()
|
||||
obj._on_deferred_stream_complete = None
|
||||
obj.model = model
|
||||
return obj
|
||||
|
||||
|
||||
|
|
@ -409,7 +410,7 @@ async def _collect_openai_passthrough_chunks(chunks, endpoint_type):
|
|||
async for chunk in PassThroughStreamingHandler.chunk_processor(
|
||||
response=response,
|
||||
request_body={"model": "gpt-4o-mini", "stream": True},
|
||||
litellm_logging_obj=_unarmed_logging_obj(),
|
||||
litellm_logging_obj=_unarmed_logging_obj(model="gpt-4o-mini"),
|
||||
endpoint_type=endpoint_type,
|
||||
start_time=datetime(2026, 1, 1),
|
||||
passthrough_success_handler_obj=MagicMock(),
|
||||
|
|
|
|||
|
|
@ -346,6 +346,7 @@ async def test_vertex_ai_anthropic_streaming_cost_injection_enabled():
|
|||
|
||||
# Setup logging object with model info
|
||||
litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj)
|
||||
litellm_logging_obj.model = "claude-sonnet-4@20250514"
|
||||
litellm_logging_obj.litellm_params = {}
|
||||
litellm_logging_obj.model_call_details = {"model": "claude-sonnet-4@20250514"}
|
||||
litellm_logging_obj.completion_start_time = None
|
||||
|
|
@ -544,6 +545,7 @@ async def test_vertex_ai_anthropic_streaming_model_extraction():
|
|||
response.aiter_bytes = mock_aiter_bytes
|
||||
|
||||
litellm_logging_obj = MagicMock(spec=LiteLLMLoggingObj)
|
||||
litellm_logging_obj.model = "claude-sonnet-4@20250514"
|
||||
litellm_logging_obj.litellm_params = {}
|
||||
litellm_logging_obj.model_call_details = {}
|
||||
litellm_logging_obj.completion_start_time = None
|
||||
|
|
|
|||
|
|
@ -74,6 +74,7 @@ from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitErro
|
|||
from litellm.proxy.common_utils.sse_keepalive import ANTHROPIC_PING_SSE_CHUNK
|
||||
from litellm.proxy.dd_span_tagger import DDSpanTagger
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.llms.anthropic.chat.transformation import AnthropicConfig
|
||||
from litellm.proxy._types import Litellm_EntityType, ProxyErrorTypes, ProxyException
|
||||
from litellm.proxy._types import UserAPIKeyAuth as ProxyUserAPIKeyAuth
|
||||
from litellm.proxy.auth.fallback_budget import RouterFallbackBudgetCheck
|
||||
|
|
@ -10221,6 +10222,8 @@ class TestInjectCostIntoUsageDict:
|
|||
discount streamed a ``usage.cost`` that disagreed with the callback's ``response_cost``."""
|
||||
|
||||
class _StubLoggingObj:
|
||||
model = "claude-haiku-4-5"
|
||||
|
||||
def __init__(self, cost):
|
||||
self._cost = cost
|
||||
self.captured_result = None
|
||||
|
|
@ -10257,6 +10260,8 @@ class TestInjectCostIntoUsageDict:
|
|||
|
||||
def test_message_delta_falls_back_to_model_pricing_when_the_logging_obj_returns_no_cost(self):
|
||||
class _StubLoggingObj:
|
||||
model = "claude-haiku-4-5"
|
||||
|
||||
def response_cost_calculator(self, result):
|
||||
return None
|
||||
|
||||
|
|
@ -10282,6 +10287,8 @@ class TestInjectCostIntoUsageDict:
|
|||
model-name pricing rather than propagating into the response body."""
|
||||
|
||||
class _StubLoggingObj:
|
||||
model = "claude-haiku-4-5"
|
||||
|
||||
def response_cost_calculator(self, result):
|
||||
raise ValueError("no pricing for this deployment")
|
||||
|
||||
|
|
@ -10363,6 +10370,8 @@ class TestInjectCostIntoUsageDict:
|
|||
streaming /v1/chat/completions gets its negotiated price instead of sticker."""
|
||||
|
||||
class _StubLoggingObj:
|
||||
model = "gpt-4o-mini"
|
||||
|
||||
def __init__(self, cost):
|
||||
self._cost = cost
|
||||
self.captured_result = None
|
||||
|
|
@ -10391,6 +10400,8 @@ class TestInjectCostIntoUsageDict:
|
|||
|
||||
def test_openai_chunk_falls_back_to_model_pricing_when_the_logging_obj_returns_no_cost(self):
|
||||
class _StubLoggingObj:
|
||||
model = "gpt-4o-mini"
|
||||
|
||||
def response_cost_calculator(self, result):
|
||||
return None
|
||||
|
||||
|
|
@ -10406,6 +10417,60 @@ class TestInjectCostIntoUsageDict:
|
|||
assert result is not None
|
||||
assert result["usage"]["cost"] == pytest.approx(self._expected_cost("gpt-4o-mini", 11, 4))
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_model", "served_model", "custom_llm_provider"),
|
||||
[
|
||||
("vertex_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "vertex_ai"),
|
||||
("anthropic/claude-sonnet-5-5", "claude-sonnet-5-5", "anthropic"),
|
||||
("azure_ai/claude-sonnet-5-5", "claude-sonnet-5-5", "azure_ai"),
|
||||
],
|
||||
)
|
||||
def test_message_delta_bills_search_like_the_logged_response_on_a_token_priced_claude_deployment(
|
||||
self, local_model_cost_map: None, deployment_model: str, served_model: str, custom_llm_provider: str
|
||||
) -> None:
|
||||
token_rates: Final = {"input_cost_per_token": 3e-6, "output_cost_per_token": 1.5e-5}
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "priced-claude",
|
||||
"litellm_params": {
|
||||
"model": deployment_model,
|
||||
"api_key": "k",
|
||||
"vertex_project": "p",
|
||||
"vertex_location": "global",
|
||||
},
|
||||
"model_info": token_rates,
|
||||
}
|
||||
]
|
||||
)
|
||||
deployment_id: Final = router.model_list[0]["model_info"]["id"]
|
||||
logging_obj: Final = LiteLLMLoggingObj(
|
||||
model=deployment_model,
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
stream=True,
|
||||
call_type="anthropic_messages",
|
||||
start_time=None,
|
||||
litellm_call_id="token-priced-claude-search",
|
||||
function_id="token-priced-claude-search",
|
||||
)
|
||||
logging_obj.update_environment_variables(
|
||||
litellm_params={"metadata": {"model_info": {"id": deployment_id, **token_rates}}}, optional_params={}
|
||||
)
|
||||
logging_obj.model_call_details["custom_llm_provider"] = custom_llm_provider
|
||||
wire_usage: Final = {"input_tokens": 1840, "output_tokens": 412, "server_tool_use": {"web_search_requests": 2}}
|
||||
event: Final = {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": wire_usage}
|
||||
|
||||
result: Final = ProxyBaseLLMRequestProcessing._inject_cost_into_usage_dict(event, "priced-claude", logging_obj)
|
||||
|
||||
stream_usage: Final = AnthropicConfig().calculate_usage(usage_object=wire_usage, reasoning_content=None)
|
||||
logged_cost: Final = logging_obj.response_cost_calculator(
|
||||
result=litellm.ModelResponse(model=served_model, usage=stream_usage)
|
||||
)
|
||||
token_cost: Final = 1840 * token_rates["input_cost_per_token"] + 412 * token_rates["output_cost_per_token"]
|
||||
assert logged_cost is not None and logged_cost > token_cost
|
||||
assert result is not None
|
||||
assert result["usage"]["cost"] == pytest.approx(logged_cost)
|
||||
|
||||
|
||||
class TestProcessChunkWithCostInjection:
|
||||
@pytest.mark.parametrize("model", ["claude-opus-5-5", "vertex_ai/claude-opus-5-5"])
|
||||
|
|
@ -10472,6 +10537,8 @@ class TestProcessChunkWithCostInjection:
|
|||
monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True)
|
||||
|
||||
class _StubLoggingObj:
|
||||
model = "claude-haiku-4-5"
|
||||
|
||||
def response_cost_calculator(self, result):
|
||||
return 0.00042
|
||||
|
||||
|
|
|
|||
|
|
@ -996,3 +996,21 @@ async def test_guardrails_only_and_skip_guardrails_are_mutually_exclusive(
|
|||
guardrails_only=True,
|
||||
skip_guardrails=True,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"request_data, expected",
|
||||
[
|
||||
({"metadata": {"_guardrail_pipelines": ["p1"]}}, True),
|
||||
({"metadata": {"user_id": "u"}, "litellm_metadata": {"_guardrail_pipelines": ["p1"]}}, True),
|
||||
({"metadata": {"_guardrail_pipelines": 5}}, True),
|
||||
({"metadata": {"user_id": "u"}}, False),
|
||||
],
|
||||
ids=["pipeline-in-metadata", "pipeline-in-litellm-metadata", "non-list-pipeline-value", "no-pipeline"],
|
||||
)
|
||||
def test_request_has_pre_call_guardrails_reads_pipelines_from_both_metadata_buckets(
|
||||
proxy_logging, monkeypatch, request_data, expected
|
||||
):
|
||||
monkeypatch.setattr(litellm, "callbacks", [])
|
||||
|
||||
assert proxy_logging.request_has_pre_call_guardrails(request_data) is expected
|
||||
|
|
|
|||
|
|
@ -6315,6 +6315,17 @@ def test_marks_past_the_provider_cap_keep_the_newest_under_drop_params(drop_para
|
|||
assert messages[1] == {"role": "user", "content": "turn 0"}
|
||||
|
||||
|
||||
def test_marks_past_the_provider_cap_inside_one_message_keep_the_newest_under_drop_params():
|
||||
parts: Final = tuple(
|
||||
{"type": "input_text", "text": f"part {index}", "cache_control": _EPHEMERAL} for index in range(1, 6)
|
||||
)
|
||||
|
||||
messages: Final = _bridged_request({"role": "user", "content": list(parts)}, drop_params=True)
|
||||
|
||||
content: Final = TypeAdapter(list[dict[str, object]]).validate_python(messages[-1]["content"])
|
||||
assert ["cache_control" in part for part in content] == [False, True, True, True, True]
|
||||
|
||||
|
||||
def test_global_drop_params_keeps_the_newest_marks(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setattr(litellm, "drop_params", True)
|
||||
|
||||
|
|
|
|||
|
|
@ -607,6 +607,23 @@ def test_openai_file_object_accepts_pending_status():
|
|||
assert file_obj.status == "pending"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("object_type", ({}, {"object": None}), ids=["missing", "null"])
|
||||
def test_openai_file_object_without_an_object_type_validates_as_a_file(object_type: dict[str, None]) -> None:
|
||||
from litellm.types.llms.openai import OpenAIFileObject
|
||||
|
||||
server_file: Final = {
|
||||
"id": "file-b73f4c",
|
||||
"bytes": 156,
|
||||
"created_at": 1793238707,
|
||||
"filename": "batch_requests.jsonl",
|
||||
"purpose": "batch",
|
||||
}
|
||||
|
||||
file_obj: Final = OpenAIFileObject.model_validate({**server_file, **object_type})
|
||||
|
||||
assert file_obj.model_dump(exclude_none=True) == {**server_file, "object": "file"}
|
||||
|
||||
|
||||
class TestOpenAIFileObjectBatchGuardrailSerialization:
|
||||
"""The proxy-only `litellm_batch_guardrail` key must reach the wire only when something set it."""
|
||||
|
||||
|
|
|
|||
16
ui/litellm-dashboard/package-lock.json
generated
16
ui/litellm-dashboard/package-lock.json
generated
|
|
@ -2037,6 +2037,13 @@
|
|||
"url": "https://opencollective.com/libvips"
|
||||
}
|
||||
},
|
||||
"node_modules/@isaacs/nonrecursive-glob-match": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.2.tgz",
|
||||
"integrity": "sha512-GJPHIgPPtFl+nLhFHyF0zZoFcOFLsZWbHFFzo+Zr+PWDXJbl9hHxb2w+9SiF2fHmLgKaXIpUzQo3SbBBZ9WObw==",
|
||||
"dev": true,
|
||||
"license": "BlueOak-1.0.0"
|
||||
},
|
||||
"node_modules/@jridgewell/gen-mapping": {
|
||||
"version": "0.3.13",
|
||||
"resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz",
|
||||
|
|
@ -9657,13 +9664,14 @@
|
|||
}
|
||||
},
|
||||
"node_modules/minimatch": {
|
||||
"version": "10.2.4",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.4.tgz",
|
||||
"integrity": "sha512-oRjTw/97aTBN0RHbYCdtF1MQfvusSIBQM0IZEgzl6426+8jSC0nF1a/GmnVLpfB9yyr6g6FTqWqiZVbxrtaCIg==",
|
||||
"version": "11.0.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.2.tgz",
|
||||
"integrity": "sha512-pWTgz5CN3tjqyZNSK2tePlSmu9IwM41q8yn8Usdqqwe9TIHh+9dEpLLjF4uApsfPEAZY71qtSiA2B2APEr+1Yw==",
|
||||
"dev": true,
|
||||
"license": "BlueOak-1.0.0",
|
||||
"dependencies": {
|
||||
"brace-expansion": "^5.0.2"
|
||||
"@isaacs/nonrecursive-glob-match": "^1.0.2",
|
||||
"brace-expansion": "^5.0.12"
|
||||
},
|
||||
"engines": {
|
||||
"node": "18 || 20 || >=22"
|
||||
|
|
|
|||
|
|
@ -117,7 +117,7 @@
|
|||
"js-yaml": "4.3.2",
|
||||
"brace-expansion": "5.0.12",
|
||||
"glob": "13.0.0",
|
||||
"minimatch": "10.2.4",
|
||||
"minimatch": "11.0.2",
|
||||
"ws": "8.21.0",
|
||||
"braces": "3.0.3",
|
||||
"axios": "1.13.6",
|
||||
|
|
|
|||
13
uv.lock
generated
13
uv.lock
generated
|
|
@ -16,6 +16,7 @@ exclude-newer-span = "P3D"
|
|||
|
||||
[options.exclude-newer-package]
|
||||
brotli = "2026-10-22T14:01:00Z"
|
||||
pypdf = "2026-10-28T17:01:00Z"
|
||||
|
||||
[manifest]
|
||||
members = [
|
||||
|
|
@ -8372,11 +8373,11 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "pyasn1"
|
||||
version = "0.6.4"
|
||||
version = "0.6.5"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/88/7e/3c0da41061a2021cac7cd524fd57092adbe5b515e8bf9005702e366802e2/pyasn1-0.6.5.tar.gz", hash = "sha256:6b11616a6bcf3a40e635f082bb232fa57b5e0dd7f517bd7ddd69ec3794508a47", size = 155152, upload-time = "2026-10-27T06:32:49.125Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/19/1f/3eefd5c84c35cae44549ce1c7f7e58f7945b34f5a6bce07a135438522073/pyasn1-0.6.5-py3-none-any.whl", hash = "sha256:4017b0fce125096b8573b39c989fdb1a4b19cf761e683398184550fea0e66587", size = 84807, upload-time = "2026-10-27T06:32:48.045Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -8869,14 +8870,14 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "pypdf"
|
||||
version = "6.21.2"
|
||||
version = "6.21.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "typing-extensions", version = "4.15.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/06/28/081f1cbcca7042441dafc8de00f48ba60853f89e5bff92c5f26477346d81/pypdf-6.21.2.tar.gz", hash = "sha256:0a3e1c5c886a1be03646c16a04827cc166c2a0575c971b26a4bf89686551c2d4", size = 7144571, upload-time = "2026-10-21T16:44:55.272Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/1e/e1/ec8f4d155c997c45917153a5521e60d6ee6389d68abdcbc8e44e564caffc/pypdf-6.21.4.tar.gz", hash = "sha256:c8817e3f888d47a087872ded69812c1ef30cacb871d6e053e0c00bbaf0bc2163", size = 7109255, upload-time = "2026-10-28T17:00:54.81Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/1b/4d/c9d2d67f204eff463237a1a8f5e25a6923b7b1043982b91fcaf104c8ef60/pypdf-6.21.2-py3-none-any.whl", hash = "sha256:1d44d1a3a200f25062adff0085b02040e4646431ac2ccac22e42679e91466084", size = 407766, upload-time = "2026-10-21T16:44:53.626Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/3f/4b/77e8ad4ac42c9d93600f40541a50e9feb3a61d2bbee2d76f659346072051/pypdf-6.21.4-py3-none-any.whl", hash = "sha256:33e4fb41ca352810eb4e9838aa0d82d6888a5f8bf682f43e99cce5faa666255c", size = 410430, upload-time = "2026-10-28T17:00:53.146Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
16
vscode-extension/package-lock.json
generated
16
vscode-extension/package-lock.json
generated
|
|
@ -644,6 +644,13 @@
|
|||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/@isaacs/nonrecursive-glob-match": {
|
||||
"version": "1.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@isaacs/nonrecursive-glob-match/-/nonrecursive-glob-match-1.0.2.tgz",
|
||||
"integrity": "sha512-GJPHIgPPtFl+nLhFHyF0zZoFcOFLsZWbHFFzo+Zr+PWDXJbl9hHxb2w+9SiF2fHmLgKaXIpUzQo3SbBBZ9WObw==",
|
||||
"dev": true,
|
||||
"license": "BlueOak-1.0.0"
|
||||
},
|
||||
"node_modules/@jridgewell/sourcemap-codec": {
|
||||
"version": "1.6.0",
|
||||
"resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz",
|
||||
|
|
@ -2700,13 +2707,14 @@
|
|||
}
|
||||
},
|
||||
"node_modules/minimatch": {
|
||||
"version": "10.2.6",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz",
|
||||
"integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==",
|
||||
"version": "11.0.2",
|
||||
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-11.0.2.tgz",
|
||||
"integrity": "sha512-pWTgz5CN3tjqyZNSK2tePlSmu9IwM41q8yn8Usdqqwe9TIHh+9dEpLLjF4uApsfPEAZY71qtSiA2B2APEr+1Yw==",
|
||||
"dev": true,
|
||||
"license": "BlueOak-1.0.0",
|
||||
"dependencies": {
|
||||
"brace-expansion": "^5.0.8"
|
||||
"@isaacs/nonrecursive-glob-match": "^1.0.2",
|
||||
"brace-expansion": "^5.0.12"
|
||||
},
|
||||
"engines": {
|
||||
"node": "18 || 20 || >=22"
|
||||
|
|
|
|||
|
|
@ -83,5 +83,8 @@
|
|||
"esbuild": "^0.28.2",
|
||||
"typescript": "^5.9.3",
|
||||
"vitest": "^4.1.12"
|
||||
},
|
||||
"overrides": {
|
||||
"minimatch": "11.0.2"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue