Merge remote-tracking branch 'origin/main' into HEAD

# Conflicts:
#	litellm/types/router.py
This commit is contained in:
mateo-berri 2026-10-31 07:01:53 -07:00
commit 90b154da10
179 changed files with 12521 additions and 1174 deletions

7
.circleci/AGENTS.md Normal file
View file

@ -0,0 +1,7 @@
# .circleci
The `integration` workflow on `main` runs from a scheduled pipeline in the CircleCI project settings, cron `9 0-8,14-23 * * *` (UTC) on `main` with no pipeline parameters. It has no `triggers:` block in this file because a workflow with one runs only on its schedule, so the API pipelines that set `run_migration_tests` or `routing_parity_base` would stop running it. If main stops getting integration runs, check that the schedule still exists and recreate it with these values
Jobs clone with the `checkout_over_https` command, never the built-in `checkout` step, because CircleCI falls back to the project's SSH checkout key without its GitHub App access, GitHub rejects that key, and the public repo needs no key. It fetches the job's commit with full history and no file contents (`--filter=blob:none`), like the built-in step did, so `scripts/path_filter.sh` can find a merge base
A step that checks out files from another commit fetches that commit with `--no-filter`, as `routing_parity` does, because the clone has no file contents and git would otherwise download each missing file in its own request, which took 210 to 896 seconds for the routing parity base

View file

@ -21,19 +21,30 @@ orbs:
win: circleci/windows@5.0 # Add Windows orb
commands:
checkout_migration_source:
checkout_over_https:
parameters:
revision:
type: string
default: ""
shell:
type: string
default: /bin/bash -eo pipefail
steps:
- run:
name: Select the requested migration test revision
name: Check out the public repository over HTTPS
shell: << parameters.shell >>
environment:
MIGRATION_SOURCE_SHA: << pipeline.parameters.migration_source_sha >>
REQUESTED_REVISION: << parameters.revision >>
command: |
revision="${MIGRATION_SOURCE_SHA:-$CIRCLE_SHA1}"
revision="${REQUESTED_REVISION:-$CIRCLE_SHA1}"
[[ "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1
git init
git remote add origin https://github.com/BerriAI/litellm.git
git fetch --depth 1 origin "$revision"
git checkout --detach FETCH_HEAD
for attempt in 1 2 3; do
git fetch --filter=blob:none origin "$revision" && git checkout --detach FETCH_HEAD && break
[[ $attempt -lt 3 ]] || exit 1
sleep 5
done
skip_if_unrelated_changes:
parameters:
category:
@ -347,7 +358,7 @@ commands:
- when:
condition: << parameters.checkout >>
steps:
- checkout
- checkout_over_https
- setup_google_dns
- install_uv
- install_rust
@ -377,7 +388,8 @@ jobs:
CARGO_HTTP_MULTIPLEXING: "false"
CARGO_NET_RETRY: "5"
steps:
- checkout
- checkout_over_https:
shell: bash.exe
- run:
name: Skip job when no windows-release-relevant files changed
shell: bash.exe
@ -451,7 +463,7 @@ jobs:
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
steps:
- checkout
- checkout_over_https
- setup_google_dns
- install_uv
- install_rust
@ -482,7 +494,7 @@ jobs:
working_directory: ~/project
steps:
- checkout
- checkout_over_https
- skip_if_unrelated_changes
- attach_workspace:
at: ~/project
@ -559,7 +571,7 @@ jobs:
resource_class: medium
working_directory: ~/project
steps:
- checkout
- checkout_over_https
- run:
name: Generate LiteLLM master key
command: |
@ -659,7 +671,7 @@ jobs:
resource_class: large
working_directory: ~/project
steps:
- checkout
- checkout_over_https
- run:
name: Generate LiteLLM master key
command: |
@ -740,7 +752,7 @@ jobs:
resource_class: large
working_directory: ~/project
steps:
- checkout
- checkout_over_https
- skip_if_unrelated_changes:
category: client
- setup_google_dns
@ -794,7 +806,7 @@ jobs:
# the rest of the suite: nothing else performs a logout.
PROXY_LOGOUT_URL: "https://www.example.com"
steps:
- checkout
- checkout_over_https
- run:
name: Generate LiteLLM master key
command: |
@ -940,7 +952,7 @@ jobs:
# and by migration.serverRootPath.config.ts, which refuses to run without it.
SERVER_ROOT_PATH: "/litellm"
steps:
- checkout
- checkout_over_https
- run:
name: Generate LiteLLM master key
command: |
@ -1066,11 +1078,12 @@ jobs:
- when:
condition: << parameters.migration_qualification >>
steps:
- checkout_migration_source
- checkout_over_https:
revision: << pipeline.parameters.migration_source_sha >>
- unless:
condition: << parameters.migration_qualification >>
steps:
- checkout
- checkout_over_https
- skip_if_unrelated_changes
- run:
@ -1122,7 +1135,8 @@ jobs:
MIGRATION_TEST_OUTPUT: /tmp/migration-results
PYTHONPATH: tests/e2e
steps:
- checkout_migration_source
- checkout_over_https:
revision: << pipeline.parameters.migration_source_sha >>
- run:
name: Generate LiteLLM master key
command: |
@ -1197,7 +1211,7 @@ jobs:
resource_class: medium
working_directory: ~/project
steps:
- checkout
- checkout_over_https
- skip_if_unrelated_changes
- attach_workspace:
at: ~/project
@ -1371,7 +1385,7 @@ jobs:
environment:
UV_PYTHON_DOWNLOADS: never
steps:
- checkout
- checkout_over_https
- install_uv
- run:
name: Install the locked shard verifier and coverage dependencies
@ -1419,7 +1433,7 @@ jobs:
ROUTING_PARITY_BASE: << pipeline.parameters.routing_parity_base >>
command: |
[[ "$ROUTING_PARITY_BASE" =~ ^[0-9a-f]{40}$ ]] || exit 1
git fetch --depth 1 origin "$ROUTING_PARITY_BASE"
git fetch --depth 1 --no-filter origin "$ROUTING_PARITY_BASE"
git rm -r -f --quiet litellm enterprise litellm-proxy-extras
git checkout "$ROUTING_PARITY_BASE" -- litellm enterprise litellm-proxy-extras
git reset --quiet
@ -1494,7 +1508,7 @@ jobs:
environment:
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm_test
steps:
- checkout
- checkout_over_https
- skip_if_unrelated_changes
- setup_litellm_test_deps:
checkout: false

View file

@ -13,6 +13,7 @@ const FINISH_REASONS: &[(&str, &str)] = &[
("max_tokens", "length"),
("refusal", "content_filter"),
("compaction", "length"),
("pause_turn", "length"),
("model_context_window_exceeded", "length"),
("guardrail_intervened", "content_filter"),
("content_filtered", "content_filter"),
@ -75,6 +76,7 @@ mod tests {
#[case::end_turn("end_turn", "stop")]
#[case::stop_sequence("stop_sequence", "stop")]
#[case::max_tokens("max_tokens", "length")]
#[case::pause_turn("pause_turn", "length")]
#[case::model_context_window_exceeded("model_context_window_exceeded", "length")]
#[case::refusal("refusal", "content_filter")]
#[case::guardrail_intervened("guardrail_intervened", "content_filter")]

View file

@ -25,7 +25,11 @@ from litellm._internal_context import current_service_target, service_target
from litellm._logging import verbose_logger
from litellm.constants import CACHED_STREAMING_CHUNK_DELAY
from litellm.integrations.otel.runtime import phase_span
from litellm.litellm_core_utils.model_param_helper import ModelParamHelper, peek_reasoning_summary_aliases
from litellm.litellm_core_utils.model_param_helper import (
ModelParamHelper,
embeds_thought_signature_in_tool_call_id,
peek_reasoning_summary_aliases,
)
from litellm.types.caching import *
from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
@ -421,7 +425,12 @@ class Cache:
reasoning_summary_material: Final = (
"" if reasoning_summary is None else f"reasoning_summary: {reasoning_summary}"
)
hashed_cache_key = Cache._get_hashed_cache_key(cache_key + reasoning_summary_material)
tool_call_id_shape_material: Final = (
"" if embeds_thought_signature_in_tool_call_id(kwargs) else "thought_signature_in_tool_call_id: False"
)
hashed_cache_key = Cache._get_hashed_cache_key(
cache_key + reasoning_summary_material + tool_call_id_shape_material
)
hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
verbose_logger.debug(
"\nCreated cache key: %s (source material length: %d)",

View file

@ -177,6 +177,8 @@ MID_STREAM_FAILURE_LOG_WAIT_SECONDS: Final = 1.0
# Set to 0 to disable truncation.
MAX_BASE64_LENGTH_FOR_LOGGING: Final = get_env_int("MAX_BASE64_LENGTH_FOR_LOGGING", 64)
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS: Final = 256 * 1024
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS: Final = 256
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS: Final = 8 * 1024 * 1024
REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
# in-memory stand-in handed to provider converters for redacted arguments; never stored
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER: Final = "{}"
@ -634,6 +636,8 @@ STS_CREDENTIAL_EXPIRY_SAFETY_MARGIN_SECONDS: Final = 60
BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = get_env_int("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024)
# Anthropic's Messages API rejects thinking.budget_tokens < 1024.
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024
ANTHROPIC_MAX_USER_ID_LENGTH: Final = 512
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH: Final = 255
ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:"
REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_float("REPLICATE_POLLING_DELAY_SECONDS", 0.5)
DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = get_env_int("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 128000)
@ -1163,6 +1167,7 @@ openai_compatible_providers: Final[list[str]] = [
"cortecs",
"scx-ai",
"prism",
"hostinger_router",
"sail",
"reka",
"streamlake",
@ -1686,6 +1691,7 @@ BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"
VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY: Final = "responses_websocket_call_id"
BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"

View file

@ -3319,7 +3319,7 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor):
)
_RESPONSES_WS_BILLABLE_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete"})
_RESPONSES_WS_BILLABLE_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete", "response.failed"})
class _ResponsesWsEventResponse(LiteLLMBaseModel):

View file

@ -3,10 +3,10 @@ from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field, replace
from functools import partial
from types import MappingProxyType
from typing import Final, Literal, TypeAlias
from typing import Annotated, Final, Literal, TypeAlias
import httpx
from pydantic import BaseModel, ConfigDict, StrictBool, TypeAdapter, ValidationError
from pydantic import BaseModel, BeforeValidator, ConfigDict, TypeAdapter, ValidationError
from pydantic_core import to_jsonable_python
from typing_extensions import assert_never
@ -23,6 +23,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.decisions.transformation import (
BaseDecisionsConfig,
DecisionsResponseValidationError,
catalog_mode,
image_media_type,
image_param,
input_images,
@ -61,7 +62,7 @@ from litellm.types.decisions import (
UnsupportedDecisionsRequest,
)
from litellm.types.llms.openai import ChatCompletionUserMessage
from litellm.types.utils import LlmProviders, ProviderSpecificHeader
from litellm.types.utils import LlmProviders, ProviderSpecificHeader, is_decisions_model_mode
from litellm.utils import ProviderConfigManager, get_model_cost_key
SystemOneQuestions: TypeAlias = Mapping[str, DecisionQuestion | Mapping[str, object]]
@ -257,19 +258,105 @@ def _request_safety_identifier(
return None
def _fields_without_unsupported_media(
fields: RequestFields,
def _bool_or_none(value: object) -> bool | None:
return value if isinstance(value, bool) else None
def _str_or_none(value: object) -> str | None:
return value if isinstance(value, str) else None
_DeploymentFlag: TypeAlias = Annotated[bool | None, BeforeValidator(_bool_or_none)]
class _DeploymentCapabilities(BaseModel):
model_config = ConfigDict(frozen=True, from_attributes=True)
mode: Annotated[str | None, BeforeValidator(_str_or_none)] = None
supports_box_questions: _DeploymentFlag = None
supports_audio_input: _DeploymentFlag = None
supports_video_input: _DeploymentFlag = None
def decisions_media_flag(self, flag: bool | None, model: str, custom_llm_provider: str) -> bool | None:
if flag is None:
return None
mode: Final = self.mode if self.mode is not None else catalog_mode(model, custom_llm_provider)
return flag if mode is None or is_decisions_model_mode(mode) else None
@dataclass(frozen=True, slots=True)
class _ResolvedSupport:
box_questions: bool
audio: bool
videos: bool
def _takes_audio(
provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities
) -> bool:
return provider_config.supports_audio_input(
model,
custom_llm_provider,
deployment.decisions_media_flag(deployment.supports_audio_input, model, custom_llm_provider),
)
def _takes_videos(
provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str, deployment: _DeploymentCapabilities
) -> bool:
return provider_config.supports_video_input(
model,
custom_llm_provider,
deployment.decisions_media_flag(deployment.supports_video_input, model, custom_llm_provider),
)
def deployment_refuses_media(
model: str,
custom_llm_provider: str,
model_info: Mapping[str, object],
call_kwargs: Mapping[str, object],
*,
audio: bool,
videos: bool,
) -> bool:
if _drops_params(call_kwargs):
return False
provider_config: Final = get_decisions_provider_config(model, custom_llm_provider)
canonical_model: Final = provider_config.canonical_model(model)
deployment: Final = _DeploymentCapabilities.model_validate(model_info)
return (audio and not _takes_audio(provider_config, canonical_model, custom_llm_provider, deployment)) or (
videos and not _takes_videos(provider_config, canonical_model, custom_llm_provider, deployment)
)
def _resolved_support(
provider_config: BaseDecisionsConfig,
model: str,
custom_llm_provider: str,
headers: Mapping[str, str],
kwargs: Mapping[str, object],
fields: RequestFields,
) -> _ResolvedSupport:
deployment: Final = _DeploymentCapabilities.model_validate(kwargs.get("model_info") or {})
return _ResolvedSupport(
box_questions=provider_config.supports_box_questions(
model, custom_llm_provider, headers, deployment.supports_box_questions
),
audio=bool(fields.audio) and _takes_audio(provider_config, model, custom_llm_provider, deployment),
videos=bool(fields.videos) and _takes_videos(provider_config, model, custom_llm_provider, deployment),
)
def _fields_without_unsupported_media(
fields: RequestFields, support: _ResolvedSupport, kwargs: Mapping[str, object]
) -> RequestFields:
if not _drops_params(kwargs):
return fields
return replace(
fields,
audio=fields.audio if provider_config.supports_audio_input(model, custom_llm_provider) else None,
videos=fields.videos if provider_config.supports_video_input(model, custom_llm_provider) else None,
audio=fields.audio if support.audio else None,
videos=fields.videos if support.videos else None,
)
@ -292,10 +379,8 @@ class _MalformedImages:
def _unsupported_params(
ir_request: DecisionsIRRequest,
provider_config: BaseDecisionsConfig,
model: str,
custom_llm_provider: str,
headers: Mapping[str, str],
supports_box_questions: bool,
support: _ResolvedSupport,
) -> tuple[str, ...]:
refuses_safety_identifier: Final = (
ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier
@ -304,13 +389,9 @@ def _unsupported_params(
None if provider_config.supports_images_in(ir_request.input, headers) else image_param(ir_request.input)
)
refuses_assets: Final = bool(ir_request.assets) and not provider_config.supports_assets(headers)
refuses_box_questions: Final = _has_box_question(ir_request.questions) and not supports_box_questions
refuses_audio: Final = bool(ir_request.audio) and not provider_config.supports_audio_input(
model, custom_llm_provider
)
refuses_videos: Final = bool(ir_request.videos) and not provider_config.supports_video_input(
model, custom_llm_provider
)
refuses_box_questions: Final = _has_box_question(ir_request.questions) and not support.box_questions
refuses_audio: Final = bool(ir_request.audio) and not support.audio
refuses_videos: Final = bool(ir_request.videos) and not support.videos
return (
*(("safety_identifier",) if refuses_safety_identifier else ()),
*(() if refused_image_param is None else (refused_image_param,)),
@ -333,7 +414,7 @@ def _without_unsupported_params(
ir_request: DecisionsIRRequest,
provider_config: BaseDecisionsConfig,
headers: Mapping[str, str],
supports_box_questions: bool,
support: _ResolvedSupport,
) -> DecisionsIRRequest:
return replace(
ir_request,
@ -345,9 +426,7 @@ def _without_unsupported_params(
safety_identifier=ir_request.safety_identifier if provider_config.supports_safety_identifier else None,
assets=ir_request.assets if provider_config.supports_assets(headers) else MappingProxyType({}),
questions=(
ir_request.questions
if supports_box_questions
else tuple(filter(_is_not_box_question, ir_request.questions))
ir_request.questions if support.box_questions else tuple(filter(_is_not_box_question, ir_request.questions))
),
)
@ -368,39 +447,20 @@ def _without_malformed_images(
return well_formed
class _DeploymentBoxQuestions(BaseModel):
model_config = ConfigDict(frozen=True, from_attributes=True)
supports_box_questions: StrictBool | None = None
def _deployment_supports_box_questions(kwargs: Mapping[str, object]) -> bool | None:
try:
return _DeploymentBoxQuestions.model_validate(kwargs.get("model_info") or {}).supports_box_questions
except ValidationError:
return None
def _provider_ir_request(
ir_request: DecisionsIRRequest,
*,
provider_config: BaseDecisionsConfig,
model: str,
custom_llm_provider: str,
headers: Mapping[str, str],
support: _ResolvedSupport,
kwargs: Mapping[str, object],
) -> DecisionsIRRequest | _UnsupportedParams | _NothingLeftAfterDrop:
supports_box_questions: Final = provider_config.supports_box_questions(
model, custom_llm_provider, headers, _deployment_supports_box_questions(kwargs)
)
unsupported: Final = _unsupported_params(
ir_request, provider_config, model, custom_llm_provider, headers, supports_box_questions
)
unsupported: Final = _unsupported_params(ir_request, provider_config, headers, support)
if not unsupported:
return ir_request
if not _drops_params(kwargs):
return _UnsupportedParams(names=unsupported)
supported: Final = _without_unsupported_params(ir_request, provider_config, headers, supports_box_questions)
supported: Final = _without_unsupported_params(ir_request, provider_config, headers, support)
if _leaves_no_input(supported.input):
return _NothingLeftAfterDrop(names=unsupported, missing="input to decide on")
if not supported.questions:
@ -479,10 +539,11 @@ def prepare_call(
raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error
try:
headers: Final = decisions_outbound_headers(extra_headers, provider, kwargs)
support: Final = _resolved_support(provider_config, canonical_model, provider, headers, kwargs, fields)
request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs)
request: Final = _fields_without_unsupported_media(
fields, provider_config, canonical_model, provider, kwargs
).validate(safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs))
request: Final = _fields_without_unsupported_media(fields, support, kwargs).validate(
safety_identifier=request_safety_identifier, drops_params=_drops_params(kwargs)
)
except ValidationError as error:
raise litellm.BadRequestError(
message=f"Invalid Decisions request: {error}",
@ -523,12 +584,7 @@ def prepare_call(
llm_provider=provider,
)
ir_request: Final = _provider_ir_request(
caller_ir_request,
provider_config=provider_config,
model=canonical_model,
custom_llm_provider=provider,
headers=headers,
kwargs=kwargs,
caller_ir_request, provider_config=provider_config, headers=headers, support=support, kwargs=kwargs
)
if isinstance(ir_request, _UnsupportedParams):
raise litellm.UnsupportedParamsError(

View file

@ -14,7 +14,7 @@ import itertools
import os
import re
from collections.abc import Callable, Container, Iterable, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
from urllib.parse import urlparse
from pydantic import TypeAdapter, ValidationError
@ -102,6 +102,192 @@ def _validated_object_list(value: object) -> list[object] | None:
return None
_MessageT: Final = TypeVar("_MessageT")
_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None]
_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls")
def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]:
lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS)
return tuple(
itertools.chain.from_iterable(
_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list)
)
)
def _block_carries_cache_control(block: object) -> bool:
return isinstance(block, dict) and _CONTENT_PART_ADAPTER.validate_python(block).get("cache_control") is not None
_ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT: Final = frozenset({"tool", "function"})
def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool:
if fields.get("cache_control") is None:
return False
return (
not isinstance(fields.get("content"), list)
or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
)
def cache_control_count(message: object) -> int:
if not isinstance(message, dict):
return 0
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block))
return int(_message_mark_reaches_provider(fields)) + block_marks
def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]:
others: Final = {key: value for key, value in fields.items() if key != "cache_control"}
if "cache_control" not in fields:
return others
cache_control: Final = rewrite(fields["cache_control"])
return others if cache_control is None else {**others, "cache_control": cache_control}
def block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object:
if not isinstance(block, dict):
return block
return _fields_with_cache_control(_CONTENT_PART_ADAPTER.validate_python(block), rewrite)
def with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT:
if not isinstance(message, dict):
return message
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
blocks_rewritten: Final = {
key: (
[block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)]
if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list)
else value
)
for key, value in fields.items()
}
return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped
_MessageT, _fields_with_cache_control(blocks_rewritten, rewrite)
)
def dropped_cache_control(_: object) -> None:
return None
def _first_marked_index(value: object) -> int | None:
if not isinstance(value, list):
return None
blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
return next((index for index, block in enumerate(blocks) if _block_carries_cache_control(block)), None)
_MarkSlot: TypeAlias = tuple[str, int]
_MESSAGE_MARK_SLOT: Final[_MarkSlot] = ("cache_control", -1)
def _block_mark_slots(fields: Mapping[str, object], key: str) -> tuple[_MarkSlot, ...]:
value: Final = fields.get(key)
if not isinstance(value, list):
return ()
blocks: Final = _OBJECT_LIST_ADAPTER.validate_python(value)
return tuple((key, index) for index, block in enumerate(blocks) if _block_carries_cache_control(block))
def _mark_slots_in_wire_order(fields: Mapping[str, object]) -> tuple[_MarkSlot, ...]:
content_slots: Final = _block_mark_slots(fields, "content")
tool_call_slots: Final = _block_mark_slots(fields, "tool_calls")
if not _message_mark_reaches_provider(fields):
return content_slots + tool_call_slots
if isinstance(fields.get("content"), list):
return content_slots + (_MESSAGE_MARK_SLOT,) + tool_call_slots
return (_MESSAGE_MARK_SLOT,) + content_slots + tool_call_slots
def _blocks_without_marks(key: str, value: object, dropped_slots: frozenset[_MarkSlot]) -> object:
if key not in _MARK_CARRYING_MESSAGE_FIELDS or not isinstance(value, list):
return value
return [
block_with_cache_control(block, dropped_cache_control) if (key, index) in dropped_slots else block
for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(value))
]
def _without_oldest_marks(message: _MessageT, dropped: int) -> _MessageT:
if dropped == 0 or not isinstance(message, dict):
return message
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
dropped_slots: Final = frozenset(_mark_slots_in_wire_order(fields)[:dropped])
blocks_unmarked: Final = {key: _blocks_without_marks(key, value, dropped_slots) for key, value in fields.items()}
return cast( # cast-ok: the same message with its oldest wire-order marks dropped
_MessageT,
_fields_with_cache_control(blocks_unmarked, dropped_cache_control)
if _MESSAGE_MARK_SLOT in dropped_slots
else blocks_unmarked,
)
def tool_message_with_single_cache_mark(message: _MessageT) -> _MessageT:
if not isinstance(message, dict):
return message
fields: Final = _CONTENT_PART_ADAPTER.validate_python(message)
content: Final = fields.get("content")
if fields.get("role") != "tool" or not isinstance(content, list):
return message
kept_index: Final = None if fields.get("cache_control") is not None else _first_marked_index(content)
return cast( # cast-ok: the same tool message keeping only the mark its single cache point reads
_MessageT,
{
**fields,
"content": [
block if index == kept_index else block_with_cache_control(block, dropped_cache_control)
for index, block in enumerate(_OBJECT_LIST_ADAPTER.validate_python(content))
],
},
)
_BudgetSpend: TypeAlias = Callable[[tuple[int, int], int], tuple[int, int]]
def _newest_marks_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
remaining, _ = state
kept: Final = min(marks, remaining)
return remaining - kept, marks - kept
def _whole_message_kept(state: tuple[int, int], marks: int) -> tuple[int, int]:
remaining, _ = state
if marks > remaining:
return remaining, marks
return remaining - marks, 0
def _messages_within_budget(
messages: Sequence[_MessageT], reserved_marks: int, spend: _BudgetSpend
) -> Sequence[_MessageT]:
budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0)
newest_first: Final = tuple(reversed(messages))
budget_after_each: Final = itertools.accumulate(
(cache_control_count(message) for message in newest_first), spend, initial=(budget, 0)
)
dropped_counts: Final = tuple(dropped for _, dropped in itertools.islice(budget_after_each, 1, None))
kept_newest_first: Final = tuple(
_without_oldest_marks(message, dropped) for message, dropped in zip(newest_first, dropped_counts)
)
return tuple(reversed(kept_newest_first))
def newest_marks_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]:
return _messages_within_budget(messages, reserved_marks, _newest_marks_kept)
def whole_messages_within_cache_control_limit(
messages: Sequence[_MessageT], reserved_marks: int
) -> Sequence[_MessageT]:
return _messages_within_budget(messages, reserved_marks, _whole_message_kept)
def _part_with_cache_control_renamed(part: object) -> object:
if not _accepts_prompt_cache_breakpoint(part):
return part
@ -1439,3 +1625,25 @@ class AnthropicCacheControlHook(CustomPromptManagement):
llm_router=None,
)
return None
def messages_within_cache_control_cap(
messages: Sequence[_MessageT], tools: object, cache_control: object = None, request_kwargs: object = None
) -> Sequence[_MessageT]:
"""Under drop_params, drop the oldest message hints the provider's cache_control cap has no room for.
Tool hints and a top-level ``cache_control`` stay on the wire, so they are reserved first.
"""
reserved_marks: Final = AnthropicCacheControlHook.count_external_cache_breakpoints(
_validated_object_list(tools), cache_control, request_kwargs
)
kept: Final = newest_marks_within_cache_control_limit(messages, reserved_marks)
dropped: Final = sum(map(cache_control_count, messages)) - sum(map(cache_control_count, kept))
if dropped > 0:
verbose_logger.warning(
"Dropped %d cache_control hint(s) from the oldest messages because drop_params is on and the "
"provider accepts at most %d cache_control blocks per request",
dropped,
MAX_CACHE_CONTROL_BLOCKS,
)
return kept

View file

@ -311,6 +311,7 @@ _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = {
"tool_use": "tool_calls",
"refusal": "content_filter",
"compaction": "length",
"pause_turn": "length",
"model_context_window_exceeded": "length",
# Cohere
"COMPLETE": "stop",

View file

@ -84,6 +84,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
"github_copilot_host",
"github_copilot_user_session",
"fireworks_forward_user_id",
"thought_signature_in_tool_call_id",
"responses_api_version",
"allowed_openai_params",
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
@ -99,6 +100,8 @@ OPTIONAL_KWARGS_KEYS: Final = (
# Backward-compatible alias for existing imports/tests.
_OPTIONAL_KWARGS_KEYS: Final = OPTIONAL_KWARGS_KEYS
SUPPLEMENTAL_KWARGS_KEYS: Final = OPTIONAL_KWARGS_KEYS | {"provider_specific_header", "extra_headers"}
_CONTROL_OPTIONS: Final = TypeAdapter(ControlOptions)
_CONTROL_OPTION_NAMES: Final = tuple(field.name for field in fields(ControlOptions))
_MAX_SHOWN_INT_BITS: Final = 64

View file

@ -208,11 +208,14 @@ class StandardBuiltInToolCostTracking:
usage=usage, response_object=response_object
)
response_model: Final = _output_item_field(response_object, "model")
if model_info is not None and resolved_usage is not None and resolved_provider is not None:
result: Final = get_cost_for_web_search_request(
custom_llm_provider=resolved_provider,
usage=resolved_usage,
model_info=model_info,
response_model=response_model if isinstance(response_model, str) else None,
lookup_model_info=StandardBuiltInToolCostTracking._safe_get_model_info,
)
if result is not None:
return result

View file

@ -3,13 +3,15 @@ import functools
import inspect
import re
import time
from collections.abc import Iterator, Mapping, Sequence
from collections.abc import Callable, Iterator, Mapping, Sequence
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final, TypeAlias, cast
from litellm._logging import format_base64_size, verbose_logger
from litellm.constants import (
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS,
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS,
BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS,
MAX_BASE64_LENGTH_FOR_LOGGING,
)
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
@ -44,6 +46,8 @@ Helper utils used for logging callbacks
# Regex matching data-URI base64 content: "data:<mime>;base64,<payload>"
# Captures: group(1)=mime_type, group(2)=base64_payload
_DATA_URI_RE: Final = re.compile(r"data:([^;,\s]{1,255});base64,([A-Za-z0-9+/=]+)")
_DATA_URI_PREFIX: Final = "data:"
_DATA_URI_BASE64_MARKER: Final = ";base64,"
# Maximum nesting depth for _truncate_base64_in_value to guard against
# pathological payloads. OpenAI message format is typically 3-4 levels deep.
@ -60,15 +64,20 @@ def _base64_data_uri_replacer(match: re.Match[str]) -> str:
return f"data:{mime_type};base64,[base64_data truncated: {size_str}]"
def _may_contain_base64_data_uri(value: str) -> bool:
return ";" in value and _DATA_URI_PREFIX in value and _DATA_URI_BASE64_MARKER in value
def _truncate_base64_in_string(value: str) -> str:
"""Replace long base64 data-URI payloads in a string with a size placeholder."""
if MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
if MAX_BASE64_LENGTH_FOR_LOGGING <= 0 or not _may_contain_base64_data_uri(value):
return value
return _DATA_URI_RE.sub(_base64_data_uri_replacer, value)
def _truncate_base64_in_value(
value: str | dict[str, object] | list[object] | None,
truncate_string: Callable[[str], str],
) -> str | dict[str, object] | list[object] | None:
"""Iteratively truncate base64 data URIs in a JSON-like value (str/list/dict).
@ -78,13 +87,13 @@ def _truncate_base64_in_value(
# Stack entries: (source_value, depth, parent_container, key_or_index)
# We mutate *copies* of dicts/lists in-place via parent references.
if isinstance(value, str):
return _truncate_base64_in_string(value)
return truncate_string(value)
if not isinstance(value, (dict, list)):
return value
# Shallow-copy the root so we don't mutate the caller's data.
root: Final = {k: v for k, v in value.items()} if isinstance(value, dict) else list(value)
stack: Final[list] = [(root, 0)]
stack: Final[list[tuple[dict[str, object] | list[object], int]]] = [(root, 0)] # mutable-ok: explicit stack
while stack:
container, depth = stack.pop()
@ -93,19 +102,19 @@ def _truncate_base64_in_value(
if isinstance(container, dict):
for k, v in container.items():
if isinstance(v, str):
container[k] = _truncate_base64_in_string(v)
container[k] = truncate_string(v)
elif isinstance(v, dict):
copy: dict | list = {ck: cv for ck, cv in v.items()}
copy: dict[str, object] | list[object] = {ck: cv for ck, cv in v.items()}
container[k] = copy
stack.append((copy, depth + 1))
elif isinstance(v, list):
copy = list(v)
container[k] = copy
stack.append((copy, depth + 1))
elif isinstance(container, list):
else:
for i, v in enumerate(container):
if isinstance(v, str):
container[i] = _truncate_base64_in_string(v)
container[i] = truncate_string(v)
elif isinstance(v, dict):
copy = {ck: cv for ck, cv in v.items()}
container[i] = copy
@ -127,8 +136,15 @@ def truncate_base64_in_messages(
"""
if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
return messages
return _copy_messages(messages, _truncate_base64_in_string)
def _copy_messages(
messages: str | dict[str, object] | list[object], # mutable-ok: same contract as truncate_base64_in_messages
truncate_string: Callable[[str], str],
) -> str | dict[str, object] | list[object] | None: # mutable-ok: same contract as truncate_base64_in_messages
try:
return _truncate_base64_in_value(messages)
return _truncate_base64_in_value(messages, truncate_string)
except Exception as e:
verbose_logger.debug("Failed to truncate base64 in messages: %s", e)
return messages
@ -151,19 +167,31 @@ def _iter_string_leaves(value: _StringTree) -> Iterator[str]:
pass
def _is_few_plain_strings(leaves: tuple[str, ...]) -> bool:
return len(leaves) < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_STRINGS and not any(
_may_contain_base64_data_uri(leaf) for leaf in leaves
)
async def truncate_base64_in_messages_async(
messages: str | list | dict | None, # mutable-ok: same contract as truncate_base64_in_messages
) -> str | list | dict | None: # mutable-ok: same contract as truncate_base64_in_messages
"""
Same result as truncate_base64_in_messages, but payloads whose string content
reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are scanned in a worker
thread so the regex pass over multi-MB base64 images does not block the event loop.
reaches BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS are copied in a worker thread,
so neither the regex pass over multi-MB base64 images nor the copy of a long
history blocks the event loop. A few strings with no data URI, under
BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS in total, are copied inline, since
checking them for data URIs and copying them costs less than the thread hop.
"""
if messages is None or MAX_BASE64_LENGTH_FOR_LOGGING <= 0:
return messages
total_chars: Final = sum(len(leaf) for leaf in _iter_string_leaves(messages))
leaves: Final = tuple(_iter_string_leaves(messages))
total_chars: Final = sum(len(leaf) for leaf in leaves)
if total_chars < BASE64_TRUNCATION_OFFLOAD_THRESHOLD_CHARS:
return truncate_base64_in_messages(messages)
if total_chars < BASE64_TRUNCATION_INLINE_COPY_MAX_CHARS and _is_few_plain_strings(leaves):
return _copy_messages(messages, lambda text: text)
return await asyncio.to_thread(truncate_base64_in_messages, messages)

View file

@ -17,6 +17,7 @@ from openai.types.responses.response_create_params import (
ResponseCreateParamsNonStreaming,
ResponseCreateParamsStreaming,
)
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.types.llms.anthropic import AnthropicMessagesRequest
@ -41,6 +42,16 @@ def peek_reasoning_summary_aliases(optional_params: Mapping[str, object]) -> obj
return None
_OPTIONAL_BOOL: Final = TypeAdapter[bool | None](bool | None)
def embeds_thought_signature_in_tool_call_id(litellm_params: Mapping[str, object]) -> bool:
try:
return _OPTIONAL_BOOL.validate_python(litellm_params.get("thought_signature_in_tool_call_id")) is not False
except ValidationError:
return True
class ModelParamHelper:
# Cached at class level — deterministic set built from static OpenAI type annotations
relevant_logging_args: ClassVar[frozenset[str]] = frozenset()

View file

@ -88,17 +88,22 @@ def handle_any_messages_to_chat_completion_str_messages_conversion(
return [{"input": str(messages)}]
def _message_with_str_content(message: AllMessageValues) -> AllMessageValues:
texts: Final = convert_content_list_to_str(message=message)
if not texts:
return message
return cast( # cast-ok: the same message with its content flattened to one string
AllMessageValues, {**message, "content": texts}
)
def handle_messages_with_content_list_to_str_conversion(
messages: list[AllMessageValues],
) -> list[AllMessageValues]:
"""
Handles messages with content list conversion
Returns new messages with each content list flattened to a string, leaving the caller's messages untouched
"""
for message in messages:
texts = convert_content_list_to_str(message=message)
if texts:
message["content"] = texts
return messages
return [_message_with_str_content(message) for message in messages]
def strip_name_from_message(message: AllMessageValues, allowed_name_roles: list[str] = ["user"]) -> AllMessageValues:

View file

@ -26,6 +26,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
anthropic_system_messages,
find_anthropic_server_tool_result,
is_anthropic_server_tool_use,
is_anthropic_tool_search_use,
message_field,
parts_of,
@ -1252,6 +1253,22 @@ def encode_tool_call_id_with_signature(tool_call_id: str, thought_signature: str
_encode_tool_call_id_with_signature = encode_tool_call_id_with_signature
def _tool_call_id_prefix(tool_call_id: object) -> str | None:
return tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)[0] if isinstance(tool_call_id, str) else None
def _called_tool_name(tool_call_id: str, tool_calls: Sequence[Mapping[str, object]]) -> str | None:
exact_matches: Final = tuple(call for call in tool_calls if call.get("id") == tool_call_id)
prefix: Final = _tool_call_id_prefix(tool_call_id)
prefix_matches: Final = tuple(
call for call in tool_calls if prefix and _tool_call_id_prefix(call.get("id")) == prefix
)
matches: Final = exact_matches or (prefix_matches if len(prefix_matches) == 1 else ())
function: Final = matches[0].get("function") if matches else None
name: Final = function.get("name") if isinstance(function, Mapping) else None
return name if isinstance(name, str) else None
def get_thought_signature_from_tool(tool: Mapping[str, object]) -> str | None:
"""Extract thought signature from tool call's provider_specific_fields.
@ -1691,16 +1708,14 @@ def convert_to_gemini_tool_call_result(
)
) is not None:
media_parts.append(media_part)
name: str | None = message.get("name", "")
# Recover name from last message with tool calls
if last_message_with_tool_calls:
tools: Final = last_message_with_tool_calls.get("tool_calls", [])
msg_tool_call_id: Final = message.get("tool_call_id", None)
for tool in tools:
prev_tool_call_id = tool.get("id", None)
if msg_tool_call_id and prev_tool_call_id and msg_tool_call_id == prev_tool_call_id:
name = tool.get("function", {}).get("name", "")
msg_tool_call_id: Final = message.get("tool_call_id", None)
called_tool_name: Final = (
_called_tool_name(msg_tool_call_id, last_message_with_tool_calls.get("tool_calls") or ())
if last_message_with_tool_calls and isinstance(msg_tool_call_id, str) and msg_tool_call_id
else None
)
name: Final[str | None] = called_tool_name if called_tool_name is not None else message.get("name", "")
# Echo the OpenAI tool_call_id on functionResponse (strip thought-signature suffix).
gemini_call_id: str | None = None
@ -2059,6 +2074,54 @@ def _placed_and_paired(
_AnthropicToolInvokeItem: TypeAlias = AnthropicMessagesToolUseParam | dict[str, Any]
def _is_server_tool_result(block: Mapping[str, object]) -> bool:
block_type: Final = block.get("type")
return isinstance(block_type, str) and block_type.endswith("_tool_result")
def _split_unrun_server_tool_uses(
groups: Sequence[Sequence[_AnthropicToolInvokeItem]],
) -> tuple[tuple[Sequence[_AnthropicToolInvokeItem], ...], tuple[_AnthropicToolInvokeItem, ...]]:
completed: Final = tuple(group for group in groups if any(_is_server_tool_result(block) for block in group))
unrun: Final = tuple(group[0] for group in groups if not any(_is_server_tool_result(block) for block in group))
return completed, unrun
def _block_id(block: Mapping[str, object]) -> object:
return block.get("id")
def _unrun_server_tool_use_ids(blocks: Sequence[Mapping[str, object]]) -> frozenset[object]:
result_ids: Final = frozenset(block.get("tool_use_id") for block in blocks if _is_server_tool_result(block))
return frozenset(
_block_id(block)
for block in blocks
if block.get("type") == "server_tool_use" and _block_id(block) not in result_ids
)
def _last_block_id_among(blocks: Sequence[Mapping[str, object]], ids: frozenset[object]) -> object:
return next(block_id for block in reversed(blocks) if (block_id := _block_id(block)) in ids)
def _paused_turn_replay(
tool_invokes: Sequence[_AnthropicToolInvokeItem],
thinking_blocks: Sequence[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
*,
is_final_message: bool,
) -> tuple[
tuple[_AnthropicToolInvokeItem, ...],
tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...],
]:
unrun_ids: Final = _unrun_server_tool_use_ids(tool_invokes)
if not unrun_ids:
return tuple(tool_invokes), tuple(thinking_blocks)
dropped_ids: Final = (
unrun_ids - frozenset((_last_block_id_among(tool_invokes, unrun_ids),)) if is_final_message else unrun_ids
)
return tuple(block for block in tool_invokes if _block_id(block) not in dropped_ids), ()
def convert_to_anthropic_tool_invoke(
tool_calls: list[ChatCompletionAssistantToolCall],
web_search_results: Sequence[object] | None = None,
@ -2141,6 +2204,15 @@ def convert_to_anthropic_tool_invoke(
anthropic_tool_invoke.append(server_tool_result)
elif is_anthropic_tool_search_use(tool_id, tool_name) and tool_id not in answered_tool_ids:
continue
elif is_anthropic_server_tool_use(tool_id) and tool_id not in answered_tool_ids:
anthropic_tool_invoke.append(
{
"type": "server_tool_use",
"id": tool_id,
"name": tool_name,
"input": tool_input,
}
)
else:
sanitized_tool_id = sanitize_anthropic_tool_use_id(tool_id)
_anthropic_tool_use_param = AnthropicMessagesToolUseParam(
@ -3109,6 +3181,30 @@ def anthropic_messages_pt(
_has_server_tool_calls = True
break
# Get web_search_results and tool_results from provider_specific_fields
# for server_tool_use reconstruction.
# Fixes: https://github.com/BerriAI/litellm/issues/17737
_provider_specific_fields_raw = assistant_content_block.get("provider_specific_fields")
_provider_specific_fields: dict[str, Any] = (
cast(dict[str, Any], _provider_specific_fields_raw)
if isinstance(_provider_specific_fields_raw, dict)
else {}
)
tool_invoke_results = (
convert_to_anthropic_tool_invoke(
assistant_tool_calls,
web_search_results=_provider_specific_fields.get("web_search_results"),
tool_results=_provider_specific_fields.get("tool_results"),
answered_tool_ids=client_answered_tool_ids,
)
if assistant_tool_calls is not None
else ()
)
turn_paused = bool(_unrun_server_tool_use_ids(tool_invoke_results))
replayed_tool_invokes, replayed_thinking_blocks = _paused_turn_replay(
tool_invoke_results, thinking_blocks or (), is_final_message=msg_i == len(messages) - 1
)
if (
thinking_blocks is not None
and _has_server_tool_calls
@ -3121,26 +3217,12 @@ def anthropic_messages_pt(
# We must preserve this interleaved order because Anthropic
# verifies thinking block signatures based on position.
# Build the tool call groups (server_tool_use + its result)
_provider_specific_fields_raw_tc = assistant_content_block.get("provider_specific_fields")
_provider_specific_fields_tc: dict[str, Any] = {}
if isinstance(_provider_specific_fields_raw_tc, dict):
_provider_specific_fields_tc = cast(dict[str, Any], _provider_specific_fields_raw_tc)
_web_search_results_tc = _provider_specific_fields_tc.get("web_search_results")
_tool_results_tc = _provider_specific_fields_tc.get("tool_results")
tool_invoke_results = convert_to_anthropic_tool_invoke(
assistant_tool_calls,
web_search_results=_web_search_results_tc,
tool_results=_tool_results_tc,
answered_tool_ids=client_answered_tool_ids,
)
# Group tool invoke results into (server_tool_use, result) pairs
# and separate regular tool_use blocks
server_tool_groups: list[list[_AnthropicToolInvokeItem]] = []
regular_tool_uses: list[_AnthropicToolInvokeItem] = []
_current_group: list[_AnthropicToolInvokeItem] = []
for item in tool_invoke_results:
for item in replayed_tool_invokes:
item_type = item.get("type", "") if isinstance(item, dict) else getattr(item, "type", "")
if item_type == "server_tool_use":
if _current_group:
@ -3154,6 +3236,9 @@ def anthropic_messages_pt(
_current_group.append(item)
if _current_group:
server_tool_groups.append(_current_group)
completed_server_tool_groups, pending_server_tool_uses = _split_unrun_server_tool_uses(
server_tool_groups
)
# Build the text block if content is a non-empty string
text_element = None
@ -3177,15 +3262,15 @@ def anthropic_messages_pt(
# Any remaining groups (after all thinking blocks) go after.
tb_idx = 0
grp_idx = 0
num_tb = len(thinking_blocks) if thinking_blocks else 0
num_grp = len(server_tool_groups)
num_tb = len(replayed_thinking_blocks)
num_grp = len(completed_server_tool_groups)
while tb_idx < num_tb or grp_idx < num_grp:
if tb_idx < num_tb and grp_idx < num_grp:
# Emit thinking block then its tool group
assistant_content.append(thinking_blocks[tb_idx])
assistant_content.append(replayed_thinking_blocks[tb_idx])
tb_idx += 1
for block in server_tool_groups[grp_idx]:
for block in completed_server_tool_groups[grp_idx]:
item_id = block.get("id") if isinstance(block, dict) else getattr(block, "id", None)
if item_id and item_id in unique_tool_ids:
continue
@ -3195,11 +3280,11 @@ def anthropic_messages_pt(
grp_idx += 1
elif tb_idx < num_tb:
# More thinking blocks than tool groups - emit before text
assistant_content.append(thinking_blocks[tb_idx])
assistant_content.append(replayed_thinking_blocks[tb_idx])
tb_idx += 1
else:
# More tool groups than thinking blocks - emit remaining
for block in server_tool_groups[grp_idx]:
for block in completed_server_tool_groups[grp_idx]:
item_id = block.get("id") if isinstance(block, dict) else getattr(block, "id", None)
if item_id and item_id in unique_tool_ids:
continue
@ -3212,8 +3297,8 @@ def anthropic_messages_pt(
if text_element is not None:
assistant_content.append(text_element)
# Add regular (non-server) tool calls at the end
for item in regular_tool_uses:
# Add regular tool calls, then a paused turn's unrun server tool call, at the end
for item in (*regular_tool_uses, *pending_server_tool_uses):
item_id = item.get("id") if isinstance(item, dict) else getattr(item, "id", None)
if item_id and item_id in unique_tool_ids:
continue
@ -3233,18 +3318,18 @@ def anthropic_messages_pt(
)
_content_list = assistant_content_block.get("content") if _content_is_list else None
if (
thinking_blocks is not None and not _content_is_list
): # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR
assistant_content.extend(thinking_blocks)
if not _content_is_list: # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR
assistant_content.extend(replayed_thinking_blocks)
if _content_is_list and _content_list is not None:
for m in _with_missing_thinking_blocks(_content_list, thinking_blocks or ()):
for m in _with_missing_thinking_blocks(_content_list, replayed_thinking_blocks):
if not isinstance(m, dict):
continue
# handle thinking blocks
if (thinking_block := _as_thinking_block(m)) is not None:
assistant_content.extend(
_replayable_thinking_blocks([thinking_block], drop_params=drop_params)
()
if turn_paused
else _replayable_thinking_blocks([thinking_block], drop_params=drop_params)
)
continue
text_block = cast(str, m.get("text", ""))
@ -3284,25 +3369,9 @@ def anthropic_messages_pt(
assistant_content.append(_anthropic_text_content_element)
if assistant_tool_calls is not None: # support assistant tool invoke conversion
# Get web_search_results and tool_results from provider_specific_fields
# for server_tool_use reconstruction.
# Fixes: https://github.com/BerriAI/litellm/issues/17737
_provider_specific_fields_raw = assistant_content_block.get("provider_specific_fields")
_provider_specific_fields: dict[str, Any] = {}
if isinstance(_provider_specific_fields_raw, dict):
_provider_specific_fields = cast(dict[str, Any], _provider_specific_fields_raw)
_web_search_results = _provider_specific_fields.get("web_search_results")
_tool_results = _provider_specific_fields.get("tool_results")
tool_invoke_results = convert_to_anthropic_tool_invoke(
assistant_tool_calls,
web_search_results=_web_search_results,
tool_results=_tool_results,
answered_tool_ids=client_answered_tool_ids,
)
# Prevent "tool_use ids must be unique" errors by filtering duplicates
# This can happen when merging history that already contains the tool calls
for item in tool_invoke_results:
for item in replayed_tool_invokes:
# tool_use items are typically dicts, but handle objects just in case
item_id = item.get("id") if isinstance(item, dict) else getattr(item, "id", None)

View file

@ -286,6 +286,10 @@ def _separate_thinking_blocks_render(message: object) -> bool:
return any(_thinking_block_renders(block) for block in parts_of(message_field(message, "thinking_blocks")))
def is_anthropic_server_tool_use(tool_id: str) -> bool:
return tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX)
def is_anthropic_tool_search_use(tool_id: str, tool_name: str) -> bool:
return tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) and tool_name in ANTHROPIC_TOOL_SEARCH_TOOL_NAMES

View file

@ -1509,11 +1509,20 @@ def _delta_reasoning_items(delta: object) -> Sequence[object]:
return cast(Sequence[object], items) if isinstance(items, list) else () # cast-ok: items forwarded verbatim
def _chunk_reasoning_items(chunk: object) -> Sequence[object]:
def _chunk_first_delta(chunk: object) -> object:
choices: Final = _field_of(chunk, "choices")
if not isinstance(choices, list) or not choices:
return ()
return _delta_reasoning_items(_field_of(cast(Sequence[object], choices)[0], "delta")) # cast-ok: untyped chunk
return None
return _field_of(cast(Sequence[object], choices)[0], "delta") # cast-ok: untyped chunk
def _chunk_reasoning_items(chunk: object) -> Sequence[object]:
return _delta_reasoning_items(_chunk_first_delta(chunk))
def _chunk_refusal(chunk: object) -> str:
refusal: Final = _field_of(_chunk_first_delta(chunk), "refusal")
return refusal if isinstance(refusal, str) else ""
_NON_TEXT_DELTA_FIELDS: Final = (
@ -1526,6 +1535,7 @@ _NON_TEXT_DELTA_FIELDS: Final = (
"audio",
"images",
"provider_specific_fields",
"refusal",
)
@ -1674,6 +1684,9 @@ def _assembled_message_fields(processor: ChunkProcessor, chunks: Sequence) -> It
yield "reasoning_content", processor.get_combined_reasoning_content(reasoning_chunks)
typed_chunks: Final = cast(Sequence[object], chunks) # cast-ok: chunks are untyped upstream
combined_refusal: Final = "".join(_chunk_refusal(chunk) for chunk in typed_chunks)
if combined_refusal:
yield "refusal", combined_refusal
reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks))
if reasoning_items:
yield "reasoning_items", list(reasoning_items)

View file

@ -9,6 +9,8 @@ from litellm.types.utils import CallTypes
from . import *
if TYPE_CHECKING:
from collections.abc import Callable
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
)
@ -30,7 +32,29 @@ def get_cost_for_google_maps_grounding_request(
return cost_per_google_maps_grounding_request(usage=usage, model_info=model_info)
def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", model_info: "ModelInfo") -> float | None:
def _with_served_model_search_pricing(
model_info: "ModelInfo",
response_model: str | None,
served_model_provider: str,
lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None",
) -> "ModelInfo":
if model_info.get("search_context_cost_per_query") is not None or response_model is None:
return model_info
served_model_info: Final = (
lookup_model_info(response_model, served_model_provider) if lookup_model_info is not None else None
)
if served_model_info is None:
return model_info
return {**model_info, "search_context_cost_per_query": served_model_info.get("search_context_cost_per_query")}
def get_cost_for_web_search_request(
custom_llm_provider: str,
usage: "Usage",
model_info: "ModelInfo",
response_model: str | None = None,
lookup_model_info: "Callable[[str, str | None], ModelInfo | None] | None" = None,
) -> float | None:
"""
Get the cost for a web search request for a given model.
@ -38,6 +62,8 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
custom_llm_provider: The custom LLM provider.
usage: The usage object.
model_info: The model info.
response_model: The model the provider response names.
lookup_model_info: Looks up a model's cost map entry, returning None when it is not mapped.
"""
if custom_llm_provider == "gemini":
from .gemini.cost_calculator import cost_per_web_search_request
@ -46,17 +72,25 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
elif custom_llm_provider == "anthropic":
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
return get_cost_for_anthropic_web_search(
model_info=_with_served_model_search_pricing(model_info, response_model, "anthropic", lookup_model_info),
usage=usage,
)
elif custom_llm_provider.startswith("vertex_ai"):
# Anthropic Claude models on Vertex AI populate server_tool_use.web_search_requests
# (same as the direct Anthropic API), not prompt_tokens_details.web_search_requests
# (which is the Gemini field). Route claude-* models to the Anthropic calculator.
model_key: Final[str] = model_info.get("key", "") if model_info else ""
if "claude" in model_key.lower():
if "claude" in (response_model or model_key).lower():
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
verbose_logger.debug("vertex_ai/claude model detected — routing web search cost to Anthropic calculator")
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
return get_cost_for_anthropic_web_search(
model_info=_with_served_model_search_pricing(
model_info, response_model, "vertex_ai", lookup_model_info
),
usage=usage,
)
from .vertex_ai.gemini.cost_calculator import (
cost_per_web_search_request as cost_per_web_search_request_vertex_ai,
@ -71,7 +105,10 @@ def get_cost_for_web_search_request(custom_llm_provider: str, usage: "Usage", mo
return None
from .anthropic.cost_calculation import get_cost_for_anthropic_web_search
return get_cost_for_anthropic_web_search(model_info=model_info, usage=usage)
return get_cost_for_anthropic_web_search(
model_info=_with_served_model_search_pricing(model_info, response_model, "azure_ai", lookup_model_info),
usage=usage,
)
elif custom_llm_provider == "perplexity":
# Perplexity handles search costs internally in its own cost calculator
# Return 0.0 to indicate costs are already accounted for

View file

@ -70,7 +70,12 @@ from litellm.types.utils import (
)
from ...base import BaseLLM
from ..common_utils import AnthropicError, process_anthropic_headers, refusal_provider_specific_fields
from ..common_utils import (
AnthropicError,
process_anthropic_headers,
refusal_params,
stop_reason_provider_specific_fields,
)
from .transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY, AnthropicConfig, JsonModeRequest
if TYPE_CHECKING:
@ -926,6 +931,17 @@ class ModelResponseIterator:
def _web_search_call_snapshot(self) -> dict[str, object]:
return dict(self._web_search_calls)
def _refresh_pending_web_search_call(self, tool_use_id: str) -> bool:
if tool_use_id not in self._web_search_calls:
return False
self._web_search_calls[tool_use_id] = build_web_search_call(
tool_use_id,
self._server_tool_inputs.get(tool_use_id, {}),
{"content": []},
status="in_progress",
)
return True
def _complete_web_search_call(self, result: dict[str, object]) -> None:
tool_use_id: Final = result.get("tool_use_id")
if not isinstance(tool_use_id, str) or tool_use_id not in self._web_search_calls:
@ -1128,6 +1144,8 @@ class ModelResponseIterator:
self._server_tool_inputs[self._current_server_tool_id] = json.loads(args)
except (json.JSONDecodeError, TypeError):
pass
if self._refresh_pending_web_search_call(self._current_server_tool_id):
provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot()
self._current_server_tool_id = None
if self.is_response_format_tool:
text = self._flush_response_format_buffer()
@ -1140,8 +1158,8 @@ class ModelResponseIterator:
# These are automatically handled by Anthropic API, we just pass them through
pass
elif type_chunk == "message_delta":
finish_reason, usage, container, refusal_fields = self._handle_message_delta(chunk)
provider_specific_fields.update(refusal_fields)
finish_reason, usage, container, stop_reason_fields = self._handle_message_delta(chunk)
provider_specific_fields.update(stop_reason_fields)
text = self._release_answer_close() + self._drain_split_answer_blocks()
if container:
provider_specific_fields["container"] = container
@ -1190,6 +1208,7 @@ class ModelResponseIterator:
)
text, tool_use = self._handle_json_mode_chunk(text=text, tool_use=tool_use)
refusal_fields: Final = refusal_params(provider_specific_fields)
returned_chunk: Final = ModelResponseStream(
choices=[
@ -1201,6 +1220,7 @@ class ModelResponseIterator:
provider_specific_fields=(provider_specific_fields if provider_specific_fields else None),
thinking_blocks=(thinking_blocks if thinking_blocks else None),
reasoning_content=reasoning_content,
**refusal_fields,
),
finish_reason=finish_reason,
)
@ -1371,7 +1391,7 @@ class ModelResponseIterator:
self._handle_usage(anthropic_usage_chunk=message_delta["usage"]) if "usage" in message_delta else None
)
container: Final = message_delta["delta"].get("container")
return finish_reason, usage, container, refusal_provider_specific_fields(message_delta["delta"])
return finish_reason, usage, container, stop_reason_provider_specific_fields(message_delta["delta"])
def _handle_accumulated_json_chunk(self, data_str: str, is_final: bool = False) -> ModelResponseStream | None:
"""

View file

@ -120,13 +120,14 @@ from litellm.utils import (
)
from ..common_utils import (
REFUSAL_STOP_FIELDS,
STOP_REASON_FIELDS,
AnthropicError,
AnthropicModelInfo,
eager_input_streaming_flag,
process_anthropic_headers,
refusal_provider_specific_fields,
refusal_params,
requires_native_compaction_beta,
stop_reason_provider_specific_fields,
strip_advisor_blocks_from_messages,
tools_marked_for_a_tool_config_point,
)
@ -167,6 +168,9 @@ _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool)
_TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_TOOLS_ADAPTER: Final = TypeAdapter(list[object])
_LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_RESPONSE_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
_PENDING_WEB_SEARCH_RESULT: Final = MappingProxyType({"content": ()})
_REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
)
@ -2254,6 +2258,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
headers=headers,
)
@staticmethod
def _messages_within_cache_control_cap(
messages: Sequence[AllMessageValues], optional_params: Mapping[str, object]
) -> Sequence[AllMessageValues]:
from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
return messages_within_cache_control_cap(
messages, optional_params.get("tools"), optional_params.get("cache_control"), optional_params
)
def transform_request(
self,
model: str,
@ -2271,9 +2285,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
anthropic_messages_pt,
)
drop_params: Final = litellm.drop_params is True or litellm_params.get("drop_params") is True
drop_params: Final = drops_unsupported_params(litellm_params.get("drop_params"))
capped_messages: Final = (
list(
self._messages_within_cache_control_cap(
messages, _REQUEST_PARAMS_ADAPTER.validate_python(optional_params)
)
)
if drop_params
else messages
)
if "tools" not in optional_params and messages is not None and has_tool_call_blocks(messages):
if "tools" not in optional_params and has_tool_call_blocks(capped_messages):
optional_params["tools"], _ = self._map_tools(add_dummy_tool(custom_llm_provider="anthropic"))
# Drop thinking param if thinking is enabled but thinking_blocks are missing
@ -2285,9 +2308,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
# Related issue: https://github.com/BerriAI/litellm/issues/18926
if (
optional_params.get("thinking") is not None
and messages is not None
and last_assistant_with_tool_calls_has_no_thinking_blocks(messages)
and not any_assistant_message_has_thinking_blocks(messages)
and last_assistant_with_tool_calls_has_no_thinking_blocks(capped_messages)
and not any_assistant_message_has_thinking_blocks(capped_messages)
):
if litellm.modify_params:
optional_params.pop("thinking", None)
@ -2332,15 +2354,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
_name_forward_map, _name_reverse_map = self._sanitize_tool_names_in_request(
optional_params=optional_params,
)
if _name_forward_map:
messages = self._rewrite_tool_names_in_messages(messages, _name_forward_map)
request_messages: Final = (
self._rewrite_tool_names_in_messages(capped_messages, _name_forward_map)
if _name_forward_map
else capped_messages
)
if _name_reverse_map and isinstance(litellm_params, dict):
litellm_params[ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY] = _name_reverse_map
# Only the leading system run becomes the top-level system prompt. A later
# system message stays in the conversation: hoisting it rewrites the cached
# prefix and re-bills the whole history at cache-write pricing (#36559).
leading_system_run, later_messages = split_leading_system_run(messages)
leading_system_run, later_messages = split_leading_system_run(request_messages)
anthropic_system_message_list: Final = self.translate_system_message(messages=list(leading_system_run))
# Handling anthropic API Prompt Caching
if len(anthropic_system_message_list) > 0:
@ -2362,7 +2387,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
except Exception as e:
raise AnthropicError(
status_code=400,
message=f"{e}\nReceived Messages={messages}",
message=f"{e}\nReceived Messages={request_messages}",
) # don't use verbose_logger.exception, if exception is raised
self.update_headers_with_optional_anthropic_beta(
@ -2897,28 +2922,24 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
web_search_results: Sequence[object],
completion_response: Mapping[str, object],
) -> list[object]:
content: Final = completion_response.get("content")
blocks: Final = content if isinstance(content, Sequence) else ()
inputs: Final = {
call_id: tool_input
for block in blocks
if isinstance(block, Mapping)
and block.get("type") == "server_tool_use"
and block.get("name") == "web_search"
and isinstance((call_id := block.get("id")), str)
and isinstance((tool_input := block.get("input")), Mapping)
blocks: Final = _RESPONSE_BLOCKS.validate_python(completion_response.get("content") or ())
results_by_id: Final = {
tool_use_id: result
for result in _RESPONSE_BLOCKS.validate_python(web_search_results)
if result.get("type") == "web_search_tool_result"
and isinstance((tool_use_id := result.get("tool_use_id")), str)
}
return [
build_web_search_call(
tool_id=tool_use_id,
tool_input=inputs.get(tool_use_id, {}),
result=result,
tool_id=call_id,
tool_input=block.get("input"),
result=results_by_id.get(call_id, _PENDING_WEB_SEARCH_RESULT),
status=None if call_id in results_by_id else "in_progress",
)
for result in web_search_results
if isinstance(result, dict)
and result.get("type") == "web_search_tool_result"
and isinstance((tool_use_id := result.get("tool_use_id")), str)
and tool_use_id in inputs
for block in blocks
if block.get("type") == "server_tool_use"
and block.get("name") == "web_search"
and isinstance((call_id := block.get("id")), str)
]
def _build_provider_specific_fields(
@ -2942,10 +2963,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if web_search_results is not None:
provider_specific_fields["web_search_results"] = web_search_results
provider_specific_fields["web_search_calls"] = self._build_web_search_calls(
web_search_results,
completion_response,
)
web_search_calls: Final = self._build_web_search_calls(web_search_results or (), completion_response)
if web_search_results is not None or web_search_calls:
provider_specific_fields["web_search_calls"] = web_search_calls
if tool_results is not None:
provider_specific_fields["tool_results"] = tool_results
@ -2967,10 +2988,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if compaction_blocks is not None:
provider_specific_fields["compaction_blocks"] = compaction_blocks
refusal_fields: Final = refusal_provider_specific_fields(
REFUSAL_STOP_FIELDS.validate_python(completion_response)
stop_reason_fields: Final = stop_reason_provider_specific_fields(
STOP_REASON_FIELDS.validate_python(completion_response)
)
return {**provider_specific_fields, **refusal_fields}
return {**provider_specific_fields, **stop_reason_fields}
def transform_parsed_response(
self,
@ -3045,22 +3066,26 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
if json_extra_content:
merged_text = merged_text + json_extra_content if merged_text else json_extra_content
refusal_fields: Final = refusal_params(STOP_REASON_FIELDS.validate_python(completion_response))
_message = litellm.Message(
tool_calls=tool_calls_for_message,
content=merged_text or None,
provider_specific_fields=provider_specific_fields,
thinking_blocks=thinking_blocks,
reasoning_content=reasoning_content,
**refusal_fields,
)
_message.provider_specific_fields = provider_specific_fields
if json_mode_message is not None:
refusal_fields: Final = refusal_provider_specific_fields(
REFUSAL_STOP_FIELDS.validate_python(completion_response)
stop_reason_fields: Final = stop_reason_provider_specific_fields(
STOP_REASON_FIELDS.validate_python(completion_response)
)
completion_response["stop_reason"] = "stop"
if refusal_fields:
json_mode_message.provider_specific_fields = dict(refusal_fields)
if stop_reason_fields:
json_mode_message.provider_specific_fields = dict(stop_reason_fields)
if "refusal" in refusal_fields:
json_mode_message.refusal = refusal_fields["refusal"]
if thinking_blocks:
json_mode_message.thinking_blocks = thinking_blocks
json_mode_message.reasoning_content = reasoning_content

View file

@ -4,6 +4,7 @@ This file contains common utils for anthropic calls.
import copy
import json
import os
import re
from collections.abc import Container, Mapping, MutableMapping, Sequence
from dataclasses import dataclass
@ -19,6 +20,8 @@ from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm.constants import (
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH,
ANTHROPIC_MAX_USER_ID_LENGTH,
DEFAULT_MODEL_CREATED_AT_TIME,
DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
@ -178,6 +181,40 @@ def tools_without_custom_field(tools: Sequence[object], hoist_defer_loading: boo
return [_tool_without_custom_field(tool, hoist_defer_loading, drop_custom) for tool in tools]
def anthropic_pass_through_api_base() -> str:
return os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
def _same_host_and_port(url: httpx.URL, other: str) -> bool:
try:
other_url: Final = httpx.URL(other)
except httpx.InvalidURL:
return False
return (url.host, url.port) == (other_url.host, other_url.port)
def anthropic_upstream_user_id_limit(url: str) -> int | None:
upstream: Final = httpx.URL(url)
if not upstream.path.removesuffix("/").endswith("/v1/messages"):
return None
if upstream.host == "api.anthropic.com":
return ANTHROPIC_MAX_USER_ID_LENGTH
return (
ANTHROPIC_COMPATIBLE_MAX_USER_ID_LENGTH
if _same_host_and_port(upstream, anthropic_pass_through_api_base())
else None
)
def _has_email_shape(token: str) -> bool:
at_sign: Final = token.find("@", 1)
return at_sign != -1 and token.find(".", at_sign + 2, -1) != -1
def anthropic_accepts_user_id(user_id: str, max_length: int) -> bool:
return len(user_id) <= max_length and user_id.isprintable() and not any(map(_has_email_shape, user_id.split()))
def is_anthropic_messages_url(url: str) -> bool:
"""Check whether a URL addresses Anthropic's Messages API."""
parsed_url: Final = urlparse(url)
@ -2481,7 +2518,7 @@ def create_anthropic_model_list_response(
}
class RefusalStopFields(TypedDict, total=False):
class StopReasonFields(TypedDict, total=False):
stop_reason: ReadOnly[object]
stop_details: ReadOnly[object]
@ -2498,16 +2535,38 @@ def _dict_or_empty(value: object) -> object:
return value if _is_dict(value) else _no_stop_fields()
REFUSAL_STOP_FIELDS: Final[TypeAdapter[RefusalStopFields]] = TypeAdapter(
Annotated[RefusalStopFields, BeforeValidator(_dict_or_empty)]
STOP_REASON_FIELDS: Final[TypeAdapter[StopReasonFields]] = TypeAdapter(
Annotated[StopReasonFields, BeforeValidator(_dict_or_empty)]
)
_STOP_REASONS_KEPT_RAW: Final = frozenset({"refusal", "pause_turn"})
def refusal_provider_specific_fields(message: Mapping[str, object]) -> Mapping[str, object]:
def stop_reason_provider_specific_fields(message: Mapping[str, object]) -> Mapping[str, object]:
stop_reason: Final = message.get("stop_reason")
stop_details: Final = message.get("stop_details")
if stop_details is not None:
return {"stop_reason": stop_reason, "stop_details": stop_details}
if stop_reason == "refusal":
if stop_reason in _STOP_REASONS_KEPT_RAW:
return {"stop_reason": stop_reason}
return {}
class _RefusalStopDetails(TypedDict, total=False):
explanation: ReadOnly[object]
_REFUSAL_STOP_DETAILS: Final[TypeAdapter[_RefusalStopDetails]] = TypeAdapter(
Annotated[_RefusalStopDetails, BeforeValidator(_dict_or_empty)]
)
class RefusalParams(TypedDict, total=False):
refusal: ReadOnly[str]
def refusal_params(message: Mapping[str, object]) -> RefusalParams:
if message.get("stop_reason") != "refusal":
return {}
explanation: Final = _REFUSAL_STOP_DETAILS.validate_python(message.get("stop_details")).get("explanation")
return {"refusal": explanation} if isinstance(explanation, str) and explanation else {}

View file

@ -33,6 +33,7 @@ from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults
from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.anthropic import (
AnthropicStopDetails,
AppliedEdit,
CompactionBlock,
ContentBlockDelta,
@ -284,9 +285,7 @@ class _CombinedChunkSplitter:
@staticmethod
def _is_combined(chunk: "ModelResponseStream") -> bool:
"""True if ``chunk`` carries response content AND a finish_reason."""
from litellm.llms.anthropic.messages.native.utils import (
openai_chat_refusal_text,
)
from litellm.llms.anthropic.messages.native.utils import refusal_reply_text
choices: Final = _optional_attr_sequence(chunk, "choices")
if not choices:
@ -302,7 +301,7 @@ class _CombinedChunkSplitter:
or _optional_attr(delta, "tool_calls")
or _optional_attr(delta, "reasoning_content")
or _optional_attr(delta, "thinking_blocks")
or openai_chat_refusal_text(delta)
or refusal_reply_text(delta)
)
_PAYLOAD_FIELD_GROUPS: "tuple[tuple[str, ...], ...]" = (
@ -659,6 +658,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
self.compaction_block = compaction_block
self.iterations_usage = iterations_usage
self._refusal_text: str = ""
self._paused_turn: bool = False
self._claude_refusal_stop_details: AnthropicStopDetails | None = None
self.sent_compaction_block: bool = False
# Per-phase flags so the compaction block's start/delta/stop events
# are emitted (and the public state machine is advanced) in
@ -953,7 +954,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
current_content_block_index=self.current_content_block_index,
applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None),
)
processed_chunk = self._with_refusal_stop_details(processed_chunk)
processed_chunk = self._with_stop_reason_fields(processed_chunk)
# Check if this is a usage chunk and we have a held stop_reason chunk
if will_merge_into_held:
@ -1188,7 +1189,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
current_content_block_index=self.current_content_block_index,
applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None),
)
processed_chunk = self._with_refusal_stop_details(processed_chunk)
processed_chunk = self._with_stop_reason_fields(processed_chunk)
# Check if this is a usage chunk and we have a held stop_reason chunk
if will_merge_into_held:
@ -1406,14 +1407,22 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
def _increment_content_block_index(self):
self.current_content_block_index += 1
def _with_refusal_stop_details(
def _with_stop_reason_fields(
self,
processed_chunk: ContentBlockDelta | MessageBlockDelta,
) -> ContentBlockDelta | MessageBlockDelta:
if processed_chunk["type"] != "message_delta" or not self._refusal_text:
if processed_chunk["type"] != "message_delta":
return processed_chunk
delta: Final = processed_chunk["delta"]
if delta.get("stop_reason") == "max_tokens":
if self._paused_turn:
paused_delta: Final[MessageDelta] = {**delta, "stop_reason": "pause_turn"}
paused_chunk: Final[MessageBlockDelta] = {**processed_chunk, "delta": paused_delta}
return paused_chunk
if delta.get("stop_reason") == "refusal" and self._claude_refusal_stop_details is not None:
claude_refusal_delta: Final[MessageDelta] = {**delta, "stop_details": self._claude_refusal_stop_details}
claude_refusal_chunk: Final[MessageBlockDelta] = {**processed_chunk, "delta": claude_refusal_delta}
return claude_refusal_chunk
if not self._refusal_text or delta.get("stop_reason") == "max_tokens":
return processed_chunk
from litellm.llms.anthropic.messages.native.utils import (
refusal_stop_details,
@ -1470,9 +1479,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
@staticmethod
def _is_blank_delta(chunk: "ModelResponseStream") -> bool:
from litellm.llms.anthropic.common_utils import is_empty_unsigned_thinking_block
from litellm.llms.anthropic.messages.native.utils import (
openai_chat_refusal_text,
)
from litellm.llms.anthropic.messages.native.utils import refusal_reply_text
choice: Final = chunk.choices[0]
if choice.finish_reason is not None:
@ -1482,7 +1489,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
return False
if getattr(delta, "content", None):
return False
if openai_chat_refusal_text(delta):
if refusal_reply_text(delta):
return False
if getattr(delta, "reasoning_content", None):
return False
@ -1497,6 +1504,18 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
return False
return True
def _record_anthropic_stop_fields(self, delta: object) -> None:
from litellm.llms.anthropic.messages.native.utils import (
anthropic_refusal_stop_details,
anthropic_stop_reason,
)
if anthropic_stop_reason(delta) == "pause_turn":
self._paused_turn = True
claude_refusal_stop_details: Final = anthropic_refusal_stop_details(delta)
if claude_refusal_stop_details is not None:
self._claude_refusal_stop_details = claude_refusal_stop_details
def _should_start_new_content_block(self, chunk: "ModelResponseStream") -> bool:
"""
Determine if we should start a new content block based on the processed chunk.
@ -1507,16 +1526,15 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
- Different content types in the response
- Specific markers in the content
"""
from litellm.llms.anthropic.messages.native.utils import (
openai_chat_refusal_text,
)
from litellm.llms.anthropic.messages.native.utils import refusal_reply_text
from .transformation import LiteLLMAnthropicMessagesAdapter
self._record_anthropic_stop_fields(chunk.choices[0].delta)
if chunk.choices[0].finish_reason is not None:
return False
refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta)
refusal_text: Final = refusal_reply_text(chunk.choices[0].delta)
if refusal_text is not None:
self._refusal_text = self._refusal_text + refusal_text

View file

@ -150,7 +150,9 @@ from litellm.llms.anthropic.messages.native.mid_conversation_system import (
is_system_role_message,
)
from litellm.llms.anthropic.messages.native.utils import (
openai_chat_refusal_text,
anthropic_refusal_stop_details,
anthropic_stop_reason,
refusal_reply_text,
refusal_stop_details,
)
from litellm.types.llms.anthropic import (
@ -398,6 +400,10 @@ class LiteLLMAnthropicMessagesAdapter:
return None
def _streamed_tool_call_signature(self, tool_call_id: str, tool_call: object) -> str | None:
id_parts: Final = tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)
return id_parts[1] if len(id_parts) > 1 else self._extract_signature_from_tool_call(tool_call)
def _extract_signature_from_tool_use_content(self, content: Mapping[str, object]) -> str | None:
"""
Extract signature from a tool_use content block's provider_specific_fields.
@ -1467,7 +1473,7 @@ class LiteLLMAnthropicMessagesAdapter:
new_content.append(
AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump()
)
if (refusal_text := openai_chat_refusal_text(choice.message)) is not None:
if (refusal_text := refusal_reply_text(choice.message)) is not None:
new_content.append(AnthropicResponseContentBlockText(type="text", text=refusal_text).model_dump())
# Handle tool calls (in parallel to text content)
if choice.message.tool_calls is not None and len(choice.message.tool_calls) > 0:
@ -1634,7 +1640,15 @@ class LiteLLMAnthropicMessagesAdapter:
tool_name_mapping=tool_name_mapping,
)
refusal_text: Final = next(
(text for choice in response.choices if (text := openai_chat_refusal_text(choice.message)) is not None),
(text for choice in response.choices if (text := refusal_reply_text(choice.message)) is not None),
None,
)
claude_refusal_stop_details: Final = next(
(
details
for choice in response.choices
if (details := anthropic_refusal_stop_details(choice.message)) is not None
),
None,
)
@ -1643,8 +1657,10 @@ class LiteLLMAnthropicMessagesAdapter:
## extract finish reason
openai_finish_reason: Final = response.choices[0].finish_reason if response.choices else "stop"
translated_finish_reason: Final = self._translate_openai_finish_reason_to_anthropic(
openai_finish_reason=openai_finish_reason
translated_finish_reason: Final = (
"pause_turn"
if response.choices and anthropic_stop_reason(response.choices[0].message) == "pause_turn"
else self._translate_openai_finish_reason_to_anthropic(openai_finish_reason=openai_finish_reason)
)
anthropic_finish_reason: Final = (
"compaction"
@ -1686,7 +1702,11 @@ class LiteLLMAnthropicMessagesAdapter:
usage=anthropic_usage,
content=anthropic_content,
stop_reason=anthropic_finish_reason,
stop_details=(refusal_stop_details(refusal_text) if anthropic_finish_reason == "refusal" else None),
stop_details=(
(claude_refusal_stop_details or refusal_stop_details(refusal_text))
if anthropic_finish_reason == "refusal"
else None
),
)
applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None
@ -1712,10 +1732,7 @@ class LiteLLMAnthropicMessagesAdapter:
):
raw_id = choice.delta.tool_calls[0].id or str(uuid.uuid4())
tool_name = choice.delta.tool_calls[0].function.name or ""
thought_sig: str | None = None
if THOUGHT_SIGNATURE_SEPARATOR in raw_id:
parts = raw_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)
thought_sig = parts[1] if len(parts) > 1 else None
thought_sig = self._streamed_tool_call_signature(raw_id, choice.delta.tool_calls[0])
tool_block: dict[str, object] = {
"type": "tool_use",
"id": normalize_anthropic_tool_use_id(raw_id),
@ -1727,7 +1744,7 @@ class LiteLLMAnthropicMessagesAdapter:
"signature": thought_sig,
}
return "tool_use", cast("ContentBlockContentBlockDict", tool_block)
elif (choice.delta.content is not None and len(choice.delta.content) > 0) or openai_chat_refusal_text(
elif (choice.delta.content is not None and len(choice.delta.content) > 0) or refusal_reply_text(
choice.delta
) is not None:
return "text", TextBlock(type="text", text="")
@ -1810,7 +1827,7 @@ class LiteLLMAnthropicMessagesAdapter:
return "thinking_delta", ContentThinkingBlockDelta(type="thinking_delta", thinking=reasoning_content)
else:
refusal_text: Final = "".join(
refusal for choice in choices if (refusal := openai_chat_refusal_text(choice.delta)) is not None
refusal for choice in choices if (refusal := refusal_reply_text(choice.delta)) is not None
)
return "text_delta", ContentTextBlockDelta(type="text_delta", text=text + refusal_text)

View file

@ -2,7 +2,7 @@ from collections.abc import Iterable, Mapping, Sequence
from functools import lru_cache
from typing import TYPE_CHECKING, Final, cast, get_type_hints
from pydantic import JsonValue, TypeAdapter, ValidationError
from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError, with_config
from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
from litellm.litellm_core_utils.core_helpers import as_str_mapping
@ -162,6 +162,34 @@ def openai_chat_refusal_text(message_or_delta: object) -> str | None:
return _mapping_str_field(getattr(message_or_delta, "provider_specific_fields", None), "refusal")
def anthropic_stop_reason(message_or_delta: object) -> str | None:
return _mapping_str_field(getattr(message_or_delta, "provider_specific_fields", None), "stop_reason")
def refusal_reply_text(message_or_delta: object) -> str | None:
if anthropic_stop_reason(message_or_delta) == "refusal":
return None
return openai_chat_refusal_text(message_or_delta)
@with_config(ConfigDict(extra="allow"))
class _AnthropicRefusalStopDetails(AnthropicStopDetails, total=False):
pass
_ANTHROPIC_REFUSAL_STOP_DETAILS: Final = TypeAdapter(_AnthropicRefusalStopDetails)
def anthropic_refusal_stop_details(message_or_delta: object) -> AnthropicStopDetails | None:
provider_specific_fields: Final = getattr(message_or_delta, "provider_specific_fields", None)
if _mapping_str_field(provider_specific_fields, "stop_reason") != "refusal":
return None
try:
return _ANTHROPIC_REFUSAL_STOP_DETAILS.validate_python(_mapping_field(provider_specific_fields, "stop_details"))
except ValidationError:
return refusal_stop_details(openai_chat_refusal_text(message_or_delta))
def _responses_message_refusal_text(item: object) -> str | None:
from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal

View file

@ -155,6 +155,8 @@ def collapse_pure_text_chunks(
continue
json_str = ev[idx + len("data:") :].strip()
if not json_str.startswith("{"):
continue
try:
data = json.loads(json_str)
except (json.JSONDecodeError, ValueError):
@ -206,6 +208,11 @@ def collapse_pure_text_chunks(
return out
def _has_non_object_data(event_str: str) -> bool:
data_start: Final = event_str.find("data:")
return data_start != -1 and not event_str[data_start + len("data:") :].lstrip().startswith("{")
def build_complete_streaming_response_legacy(
all_chunks: Sequence[str | bytes],
litellm_logging_obj: LiteLLMLoggingObj,
@ -243,6 +250,8 @@ def build_complete_streaming_response_legacy(
# text payload happens to contain "[DONE]" is not dropped.
if any(line.strip() == "data: [DONE]" for line in event_str.split("\n")):
continue
if _has_non_object_data(event_str):
continue
transformed_openai_chunk = anthropic_model_response_iterator.convert_str_chunk_to_generic_chunk(
chunk=event_str
)
@ -268,21 +277,25 @@ def build_complete_streaming_response_legacy(
return complete_streaming_response
def extract_sse_data(event_str: str) -> dict | None:
"""Parse the JSON object from the ``data:`` line of an Anthropic SSE event."""
def _sse_data_payload(event_str: str) -> str | None:
for line in split_sse_lines(event_str):
stripped = line.strip()
if stripped.startswith("data:"):
payload = stripped[len("data:") :].strip()
if not payload or payload == "[DONE]":
return None
try:
return json.loads(payload)
except (ValueError, TypeError):
return None
return stripped[len("data:") :].strip()
return None
def extract_sse_data(event_str: str) -> dict | None:
"""Parse the JSON object from the ``data:`` line of an Anthropic SSE event."""
payload: Final = _sse_data_payload(event_str)
if payload is None or not payload.startswith("{"):
return None
try:
return json.loads(payload)
except (ValueError, TypeError):
return None
def build_usage_only_response_from_chunks(
all_chunks: Sequence[str | bytes],
model: str,
@ -415,27 +428,81 @@ _ANTHROPIC_EVENT_TYPES: Final = frozenset(
)
_ERROR_PAYLOAD_KEYS: Final = frozenset({"type", "error", "request_id"})
def _is_anthropic_event_type(event_type: object) -> bool:
return isinstance(event_type, str) and event_type in _ANTHROPIC_EVENT_TYPES
def is_raw_sse_stream(all_chunks: Sequence[object]) -> bool:
return any(isinstance(chunk, (str, bytes)) for chunk in all_chunks)
def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None:
raw: Final = b"".join(
def _joined_sse_bytes(all_chunks: Sequence[object]) -> bytes:
return b"".join(
chunk if isinstance(chunk, bytes) else chunk.encode("utf-8")
for chunk in all_chunks
if isinstance(chunk, (str, bytes))
)
def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None:
try:
return codecs.getincrementaldecoder("utf-8")().decode(raw, final=False)
return codecs.getincrementaldecoder("utf-8")().decode(_joined_sse_bytes(all_chunks), final=False)
except UnicodeDecodeError:
return None
def _sse_event_data(event: str) -> Mapping[str, object] | None:
return as_str_mapping(extract_sse_data(event))
_JSON_ESCAPED_LINE_BREAKS: Final = MappingProxyType(
str.maketrans({"\x85": "\\u0085", "\u2028": "\\u2028", "\u2029": "\\u2029"})
)
def _sse_frames(sse_stream: str) -> Sequence[str]:
return "\n".join(split_sse_lines(sse_stream)).split("\n\n")
def _frames_of_the_widest_reader(sse_stream: str) -> Sequence[str]:
"""The stream's frames as httpx's ``str.splitlines`` reads them, past a form feed or U+2028 in any line."""
return "\n".join(sse_stream.splitlines()).split("\n\n")
def _sse_field(line: str) -> tuple[str, str]:
name, _, value = line.partition(":")
return name, value.removeprefix(" ")
def _sse_fields(frame: str) -> tuple[tuple[str, str], ...]:
"""The frame's fields as a client reads them: a name matches exactly and a comment is no field."""
return tuple(_sse_field(line) for line in frame.split("\n") if line and not line.startswith(":"))
def _carries_no_data(fields: Sequence[tuple[str, str]]) -> bool:
match fields:
case () | (("event", _),) | (("data", "[DONE]"),):
return True
case _:
return False
def _json_mapping(payload: str) -> Mapping[str, object] | None:
try:
value: Final[object] = json.loads(payload)
except ValueError:
return None
return as_str_mapping(value)
def _parsed_sse_events(sse_stream: str) -> tuple[Mapping[str, object], ...]:
return tuple(
event_data
for event in split_sse_chunk_into_events(sse_stream)
if (event_data := extract_sse_data(event)) is not None
if (event_data := _sse_event_data(event)) is not None
)
@ -444,7 +511,8 @@ def _anthropic_message_start(sse_stream: str) -> Mapping[str, object] | None:
(
message
for event_data in _parsed_sse_events(sse_stream)
if event_data.get("type") == "message_start" and isinstance(message := event_data.get("message"), dict)
if event_data.get("type") == "message_start"
and (message := as_str_mapping(event_data.get("message"))) is not None
),
None,
)
@ -482,11 +550,151 @@ def is_anthropic_sse_stream(all_chunks: Sequence[object]) -> bool:
sse_stream: Final = _joined_sse_stream(all_chunks)
if sse_stream is None:
return False
return any(event.get("type") in _ANTHROPIC_EVENT_TYPES for event in _parsed_sse_events(sse_stream))
return any(_is_anthropic_event_type(event.get("type")) for event in _parsed_sse_events(sse_stream))
def _is_json_no_reader_can_take(payload: str) -> bool:
try:
value: Final[object] = json.loads(payload)
except ValueError:
return False
if value is None:
return False
event: Final = as_str_mapping(value)
return event is None or isinstance(event.get("type"), (list, dict))
def has_unreadable_json_event(all_chunks: Sequence[object]) -> bool:
"""Whether a frame's data is JSON that no reader of these streams can take as an event."""
sse_stream: Final = codecs.getincrementaldecoder("utf-8")(errors="replace").decode(
_joined_sse_bytes(all_chunks), final=False
)
return any(_is_json_no_reader_can_take(_sse_data_payload(event) or "") for event in _sse_frames(sse_stream))
def anthropic_sse_event_type(event: str) -> object:
return next((event_data.get("type") for event_data in _parsed_sse_events(event)), None)
def anthropic_sse_event_types(all_chunks: Sequence[object]) -> frozenset[str]:
"""The string ``type`` of every event the stream sent."""
event_types: Final = (event.get("type") for event in _parsed_sse_events(_joined_sse_stream(all_chunks) or ""))
return frozenset(event_type for event_type in event_types if isinstance(event_type, str))
def _is_named_error_event(event: str) -> bool:
names: Final = tuple(value for name, value in _sse_fields(event) if name == "event")
return names[-1:] == ("error",)
def _is_error_event(event: str) -> bool:
event_type: Final = anthropic_sse_event_type(event)
return event_type == "error" or (_is_named_error_event(event) and not _is_anthropic_event_type(event_type))
def _frames_a_client_dispatches(sse_stream: str) -> Sequence[str]:
return "\n".join((*split_sse_lines(sse_stream), "")).split("\n\n")[:-1]
def _error_events(sse_stream: str) -> tuple[str, ...]:
return tuple(event for event in _frames_a_client_dispatches(sse_stream) if _is_error_event(event))
def _plain_error_event(event: str) -> str:
payload: Final = _sse_data_payload(event) or ""
one_line: Final = payload.translate(_JSON_ESCAPED_LINE_BREAKS) if _json_mapping(payload) is not None else payload
return "\n".join(("event: error", *(f"data: {line}" for line in one_line.splitlines()[:1])))
def anthropic_sse_error_events(all_chunks: Sequence[object]) -> tuple[str, ...]:
"""The ``error`` events of the stream, each as an ``event:`` line and the ``data:`` line assembly read.
A stream whose upstream fails after it started ends with an ``error`` event in place of
``message_stop``. That frame is the outcome the client is owed, not content, so assembly through
the upstream error leaves it out and a caller re-emitting a rewritten response puts it back,
without any other line of the frame, since that is content no guardrail read.
A frame counts by its data ``type``, or by its SSE ``event:`` name unless its data is another
Anthropic event, since a client SDK raises on the name alone and an upstream that is not
Anthropic may send anything as the data.
"""
return tuple(_plain_error_event(event) for event in _error_events(_joined_sse_stream(all_chunks) or ""))
def anthropic_sse_stopped_block_index(event: str) -> object:
return next(
(
event_data.get("index")
for event_data in _parsed_sse_events(event)
if event_data.get("type") == "content_block_stop"
),
None,
)
def _block_index(event_data: Mapping[str, object]) -> int | None:
index: Final = event_data.get("index")
return index if isinstance(index, int) else None
def _started_blocks(events: Sequence[Mapping[str, object]]) -> Mapping[int, Mapping[str, object]]:
starts: Final = (event for event in events if event.get("type") == "content_block_start")
return MappingProxyType(
{
index: block
for event in starts
if (index := _block_index(event)) is not None
and (block := as_str_mapping(event.get("content_block"))) is not None
}
)
def anthropic_sse_started_blocks(all_chunks: Sequence[object]) -> Mapping[int, Mapping[str, object]]:
"""The ``content_block`` each ``content_block_start`` of the stream opened, by block index."""
return _started_blocks(_parsed_sse_events(_joined_sse_stream(all_chunks) or ""))
def _unfinished_blocks(events: Sequence[Mapping[str, object]]) -> Mapping[int, Mapping[str, object]]:
stopped: Final = frozenset(_block_index(event) for event in events if event.get("type") == "content_block_stop")
return MappingProxyType({index: block for index, block in _started_blocks(events).items() if index not in stopped})
def anthropic_sse_unfinished_blocks(all_chunks: Sequence[object]) -> Mapping[int, Mapping[str, object]]:
"""The blocks the stream opened and never stopped, which only a failed upstream leaves."""
return _unfinished_blocks(_parsed_sse_events(_joined_sse_stream(all_chunks) or ""))
def _sse_events_with_data(sse_stream: str) -> tuple[tuple[str, Mapping[str, object]], ...]:
return tuple(
(event, event_data)
for event in split_sse_chunk_into_events(sse_stream)
if (event_data := _sse_event_data(event)) is not None
)
def _content_events_before_upstream_error(sse_stream: str) -> tuple[str, ...]:
"""The stream's events minus its upstream ``error`` and every cut-off block assembly cannot read.
A cut-off text block is still text the client was sent, so it stays. A cut-off tool call's
input is a JSON prefix, and a cut-off thinking block has no signature, so neither is a block the
client can act on, and both are left out.
"""
if not _error_events(sse_stream):
return (sse_stream,)
events: Final = _sse_events_with_data(sse_stream)
cut_off: Final = frozenset(
index
for index, block in _unfinished_blocks(tuple(event_data for _, event_data in events)).items()
if block.get("type") != "text"
)
return tuple(
event
for event, event_data in events
if event_data.get("type") != "error" and _block_index(event_data) not in cut_off
)
def assemble_anthropic_sse_stream(
all_chunks: Sequence[object], *, restore_identity: bool = False
all_chunks: Sequence[object], *, restore_identity: bool = False, through_upstream_error: bool = False
) -> ModelResponse | None:
"""Assemble raw Anthropic SSE frames into a ModelResponse.
@ -494,6 +702,11 @@ def assemble_anthropic_sse_stream(
assembler does not carry through. It is off by default so callers that re-emit the assembled
response keep the wire shape they had before this helper was shared. The writes land on a
freshly built object that is unreachable from caller state until returned.
A stream whose upstream failed after it started ends with an ``error`` event in place of
``message_stop`` and assembles to None, unless ``through_upstream_error`` asks for the content
the client was sent before the failure. That is for a guardrail that scans the stream and then
passes the upstream error on; a caller recording what was served keeps None.
"""
sse_stream: Final = _joined_sse_stream(all_chunks)
if sse_stream is None:
@ -502,9 +715,12 @@ def assemble_anthropic_sse_stream(
if message_start is None:
return None
model: Final = message_start.get("model") if restore_identity else None
content_events: Final = (
_content_events_before_upstream_error(sse_stream) if through_upstream_error else (sse_stream,)
)
try:
assembled: Final = build_complete_streaming_response(
all_chunks=(sse_stream,),
all_chunks=content_events,
litellm_logging_obj=None, # pyright: ignore[reportArgumentType] # only forwarded to stream_chunk_builder, which accepts None
model=model if isinstance(model, str) else "",
)
@ -522,6 +738,21 @@ def assemble_anthropic_sse_stream(
return assembled
def _is_error_payload(payload: str) -> bool:
event: Final = _json_mapping(payload)
if event is None or not _ERROR_PAYLOAD_KEYS.issuperset(event):
return False
return event.get("type") == "error" or ("type" not in event and isinstance(event.get("error"), Mapping))
def _is_error_frame(fields: Sequence[tuple[str, str]]) -> bool:
match fields:
case (("data", payload),) | (("event", "error"), ("data", payload)):
return _is_error_payload(payload)
case _:
return False
def is_sse_error_stream(all_chunks: Sequence[object]) -> bool:
"""Whether the buffered stream carries nothing but error frames.
@ -537,7 +768,6 @@ def is_sse_error_stream(all_chunks: Sequence[object]) -> bool:
sse_stream: Final = _joined_sse_stream(all_chunks)
if sse_stream is None:
return False
events: Final = _parsed_sse_events(sse_stream)
return len(events) > 0 and all(
event.get("type") == "error" or isinstance(event.get("error"), Mapping) for event in events
)
frames: Final = tuple(_sse_fields(frame) for frame in _frames_of_the_widest_reader(sse_stream))
with_data: Final = tuple(fields for fields in frames if not _carries_no_data(fields))
return len(with_data) > 0 and all(_is_error_frame(fields) for fields in with_data)

View file

@ -12,6 +12,7 @@ from pydantic_core import ErrorDetails
from typing_extensions import assert_never
import litellm
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.secret_managers.main import get_secret_str
@ -297,13 +298,7 @@ def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]:
assert_never(question)
def ir_to_systemone_request(
model: str, request: DecisionsIRRequest, *, supports_audio: bool = False, supports_video: bool = False
) -> Mapping[str, object] | UnsupportedDecisionsRequest:
if request.audio and not supports_audio:
return UnsupportedDecisionsRequest(reason="audio input is not supported by this Decisions model")
if request.videos and not supports_video:
return UnsupportedDecisionsRequest(reason="video input is not supported by this Decisions model")
def ir_to_systemone_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object]:
state: Final = _systemone_state(request.input)
keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True)
images: Final = input_images(request.input)
@ -503,6 +498,16 @@ def ir_to_systemone_response(response: DecisionsIRResponse, request: DecisionsIR
)
def catalog_mode(model: str, custom_llm_provider: str) -> str | None:
try:
key: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)["key"]
except Exception: # noqa: BLE001 # get_model_info raises bare Exception for an unmapped model, which the catalog does not list
return None
entry: Final = GetModelCostMap.loaded_model_cost_map().get(key)
mode: Final = entry.get("mode") if entry is not None else None
return mode if isinstance(mode, str) else None
def _is_decisions_model(model: str, custom_llm_provider: str) -> bool:
try:
model_mode: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider).get("mode")
@ -640,12 +645,16 @@ class BaseDecisionsConfig(ABC):
model=model, custom_llm_provider=custom_llm_provider
)
def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
def supports_audio_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
if deployment_flag is not None:
return deployment_flag
return _is_decisions_model(model, custom_llm_provider) and litellm.supports_audio_input(
model=model, custom_llm_provider=custom_llm_provider
)
def supports_video_input(self, model: str, custom_llm_provider: str) -> bool:
def supports_video_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
if deployment_flag is not None:
return deployment_flag
return _is_decisions_model(model, custom_llm_provider) and litellm.supports_video_input(
model=model, custom_llm_provider=custom_llm_provider
)
@ -656,12 +665,7 @@ class BaseDecisionsConfig(ABC):
request: DecisionsIRRequest,
custom_llm_provider: str,
) -> Mapping[str, object] | UnsupportedDecisionsRequest:
return ir_to_systemone_request(
self.request_model(model),
request,
supports_audio=self.supports_audio_input(model, custom_llm_provider),
supports_video=self.supports_video_input(model, custom_llm_provider),
)
return ir_to_systemone_request(self.request_model(model), request)
def unwrap_response(self, payload: object) -> object:
return payload

View file

@ -25,6 +25,7 @@ from litellm.constants import (
)
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
drops_unsupported_params,
filter_exceptions_from_params,
filter_internal_params,
map_finish_reason,
@ -2129,6 +2130,24 @@ class AmazonConverseConfig(BaseConfig):
return data
@staticmethod
def _claude_messages_within_cache_control_cap(
messages: Sequence[AllMessageValues],
capability_model: str,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
) -> Sequence[AllMessageValues]:
if not drops_unsupported_params(litellm_params.get("drop_params")) or "claude" not in capability_model.lower():
return messages
from litellm.integrations.anthropic_cache_control_hook import (
messages_within_cache_control_cap,
tool_message_with_single_cache_mark,
)
return messages_within_cache_control_cap(
tuple(map(tool_message_with_single_cache_mark, messages)), optional_params.get("tools")
)
async def _async_transform_request(
self,
model: str,
@ -2140,7 +2159,13 @@ class AmazonConverseConfig(BaseConfig):
base_model: Final = litellm_params.get("base_model")
capability_model: Final = bedrock_capability_model(model, base_model)
messages, system_content_blocks = self._transform_system_message(
messages, model=capability_model, deployment_model=model
list(
self._claude_messages_within_cache_control_cap(
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
)
),
model=capability_model,
deployment_model=model,
)
# Convert last user message to guarded_text if guardrailConfig is present
@ -2221,7 +2246,13 @@ class AmazonConverseConfig(BaseConfig):
base_model: Final = litellm_params.get("base_model")
capability_model: Final = bedrock_capability_model(model, base_model)
messages, system_content_blocks = self._transform_system_message(
messages, model=capability_model, deployment_model=model
list(
self._claude_messages_within_cache_control_cap(
messages, capability_model, _JSON_OBJECT.validate_python(optional_params), litellm_params
)
),
model=capability_model,
deployment_model=model,
)
# Convert last user message to guarded_text if guardrailConfig is present

View file

@ -131,6 +131,8 @@ from litellm.responses.streaming_iterator import (
ResponsesAPIStreamingIterator,
ResponsesWebSocketStreaming,
SyncResponsesAPIStreamingIterator,
TokenLimitCallback,
with_token_limit_checks,
)
from litellm.types.containers.main import (
ContainerFileListResponse,
@ -598,10 +600,10 @@ def _decoded_body_headers(response: httpx.Response) -> httpx.Headers:
)
def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
"""Duck-type discover proxy hooks exposing per-frame project ITPM/OTPM
enforcement, so the Responses WebSocket loop can charge every
``response.create`` frame, not just the connection's first one.
def _ws_callbacks_exposing(method_name: str) -> tuple[object, ...]:
"""Duck-type discover proxy hooks exposing a per-frame check, so the
Responses WebSocket loop can run it on every ``response.create`` frame,
not just the connection's first one.
Uses duck-typing on ``litellm.callbacks`` (rather than importing the
proxy hook directly) to avoid a layering violation (SDK importing from
@ -612,13 +614,28 @@ def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
callbacks: Final = cast( # cast-ok: callback registry is inspected before protocol use
Sequence[object], _litellm.callbacks
)
return tuple(callback for callback in callbacks if callable(getattr(callback, method_name, None)))
def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
return tuple(
cast(ProjectQuotaCallback, callback) # cast-ok: required callback method is callable
for callback in callbacks
if callable(getattr(callback, "enforce_project_io_token_quota_for_frame", None))
for callback in _ws_callbacks_exposing("enforce_project_io_token_quota_for_frame")
)
def _collect_ws_token_limit_callbacks() -> tuple[TokenLimitCallback, ...]:
return tuple(
cast(TokenLimitCallback, callback) # cast-ok: required callback method is callable
for callback in _ws_callbacks_exposing("enforce_token_limits_for_frame")
)
def _ws_model_group(litellm_metadata: Mapping[str, object] | None, model: str) -> str:
model_group: Final = (litellm_metadata or {}).get("model_group")
return model_group if isinstance(model_group, str) and model_group else model
def _unwrap_provider_error(error: Exception) -> Exception:
if not isinstance(error, httpx.TransportError):
return error
@ -7285,6 +7302,12 @@ class BaseLLMHTTPHandler:
- Forwards events over the websocket connection
"""
_ws_quota_callbacks: Final = _collect_ws_project_quota_callbacks()
_ws_turn_admission: Final = with_token_limit_checks(
turn_admission,
_collect_ws_token_limit_callbacks(),
user_api_key_dict,
_ws_model_group(litellm_metadata, model),
)
_ws_guardrail_callbacks: list = []
_ws_output_guardrail_callbacks: list = []
@ -7332,7 +7355,7 @@ class BaseLLMHTTPHandler:
custom_llm_provider=custom_llm_provider,
first_message=first_message,
quota_callbacks=_ws_quota_callbacks,
turn_admission=turn_admission,
turn_admission=_ws_turn_admission,
guardrail_callbacks=_ws_guardrail_callbacks, # pyright: ignore[reportUnknownArgumentType] # duck-typed callbacks from litellm.callbacks
frame_guard=frame_guard,
**kwargs,
@ -7429,7 +7452,7 @@ class BaseLLMHTTPHandler:
authorized_model=model,
custom_llm_provider=custom_llm_provider,
request_defaults=request_defaults,
turn_admission=turn_admission,
turn_admission=_ws_turn_admission,
frame_guard=frame_guard,
)
return await streaming.bidirectional_forward()

View file

@ -26,6 +26,7 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase):
aembedding=None,
custom_endpoint: bool | None = None,
headers: dict | None = None,
drop_params: bool = False,
) -> EmbeddingResponse:
# Check for custom user agent in optional_params or environment
# This allows partners building on LiteLLM to set their own telemetry
@ -58,4 +59,5 @@ class DatabricksEmbeddingHandler(OpenAILikeEmbeddingHandler, DatabricksBase):
aembedding=aembedding,
custom_endpoint=True,
headers=headers,
drop_params=drop_params,
)

View file

@ -130,7 +130,8 @@ class DeepInfraConfig(OpenAIGPTConfig):
- Input: {"role": "tool", "content": [{"type": "text", "text": "result"}]}
- Output: {"role": "tool", "content": "[{\"type\": \"text\", \"text\": \"result\"}]"}
"""
for message in messages:
copied_messages: Final = [message.copy() for message in messages]
for message in copied_messages:
if message.get("role") == "tool":
content = message.get("content")
@ -149,7 +150,7 @@ class DeepInfraConfig(OpenAIGPTConfig):
# For complex content, serialize the entire array as JSON string
message["content"] = json.dumps(content)
return messages
return copied_messages
@overload
def _transform_messages(

View file

@ -23,6 +23,7 @@ from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionToolParam,
ChatCompletionToolParamFunctionChunk,
OpenAIChatCompletionToolParam,
)
from litellm.types.utils import (
@ -45,6 +46,7 @@ from ...openai.chat.gpt_transformation import (
OpenAIChatCompletionStreamingHandler,
OpenAIGPTConfig,
)
from ...openai.common_utils import without_cache_control
from ..common_utils import (
FIREROUTER,
FireworksAIException,
@ -480,15 +482,23 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
return (("response_format", _json_schema_response_format(choice_schema, "choice")),)
def _transform_tools(self, tools: list[OpenAIChatCompletionToolParam]) -> list[OpenAIChatCompletionToolParam]:
for tool in tools:
if tool.get("type") != "function":
continue
function = tool["function"]
function.pop("strict", None)
params = function.get("parameters")
if isinstance(params, dict):
unpack_legacy_defs(params)
return tools
return [self._fireworks_tool(tool) for tool in tools]
@staticmethod
def _fireworks_tool(tool: OpenAIChatCompletionToolParam) -> OpenAIChatCompletionToolParam:
if tool.get("type") != "function":
return tool
function: Final = tool["function"]
params: Final = function.get("parameters")
unstrict_function: Final = cast( # cast-ok: the same function definition minus `strict`
ChatCompletionToolParamFunctionChunk, {key: value for key, value in function.items() if key != "strict"}
)
new_function: Final[ChatCompletionToolParamFunctionChunk] = (
{**unstrict_function, "parameters": unpack_legacy_defs(params, copy=True)}
if isinstance(params, dict)
else unstrict_function
)
return {**tool, "function": new_function}
def _transform_messages_helper(
self, messages: list[AllMessageValues], model: str, litellm_params: dict
@ -496,10 +506,6 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
"""
Strip fields not permitted by FireworksAI from messages.
"""
from litellm.litellm_core_utils.prompt_templates.common_utils import (
filter_value_from_dict,
)
supports_vision_value: Final = self._get_model_cost_capability_exact(model=model, capability="supports_vision")
for message in messages:
if message["role"] == "user":
@ -530,9 +536,11 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
model=model,
llm_provider="fireworks_ai",
)
filter_value_from_dict(cast(dict, message), "cache_control")
return [strip_litellm_response_only_fields(message) for message in messages]
return [
strip_litellm_response_only_fields(cast(AllMessageValues, without_cache_control(message)))
for message in messages
]
# Cached index of fireworks_ai/* entries from litellm.model_cost. Building
# this index requires a full scan of model_cost (tens of thousands of

View file

@ -140,7 +140,8 @@ class GroqChatConfig(OpenAILikeChatConfig):
def _transform_messages(
self, messages: list[AllMessageValues], model: str, is_async: bool = False
) -> list[AllMessageValues] | Coroutine[object, object, list[AllMessageValues]]:
for idx, message in enumerate(messages):
transformed_messages: Final = list(messages)
for idx, message in enumerate(transformed_messages):
"""
1. Don't pass 'null' function_call assistant message to groq - https://github.com/BerriAI/litellm/issues/5839
"""
@ -154,12 +155,12 @@ class GroqChatConfig(OpenAILikeChatConfig):
for k, v in _message.items():
if v is not None:
new_message[k] = v
messages[idx] = strip_litellm_internal_message_fields(new_message)
transformed_messages[idx] = strip_litellm_internal_message_fields(new_message)
if is_async:
return super()._transform_messages(messages=messages, model=model, is_async=True)
return super()._transform_messages(messages=transformed_messages, model=model, is_async=True)
else:
return super()._transform_messages(messages=messages, model=model, is_async=False)
return super()._transform_messages(messages=transformed_messages, model=model, is_async=False)
def transform_messages(
self,

View file

@ -2,6 +2,7 @@
Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
"""
import copy
import json
from collections.abc import Coroutine
from typing import Final, Literal, cast, overload
@ -88,7 +89,7 @@ class HostedVLLMChatConfig(OpenAIGPTConfig):
) -> dict:
_tools = non_default_params.pop("tools", None)
if _tools is not None:
_tools = remove_additional_properties(_tools)
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
_tools = remove_strict_from_schema(_tools)
if isinstance(_tools, list):
_tools = self._convert_custom_tools_to_function_tools(_tools)

View file

@ -755,7 +755,8 @@ class MistralConfig(OpenAIGPTConfig):
if _name is not None:
# Remove name if not a tool message
if message["role"] != "tool" or isinstance(_name, str) and len(_name.strip()) == 0:
message.pop("name", None)
message_without_name: Final = {k: v for k, v in message.items() if k != "name"}
return cast(AllMessageValues, message_without_name) # cast-ok: the same message minus name
return message

View file

@ -14,7 +14,7 @@ from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.constants import OPENAI_SYSTEM_MESSAGES_FIRST_PROVIDERS
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
extract_reasoning_content,
handle_invalid_parallel_tool_calls,
@ -64,7 +64,7 @@ from litellm.types.utils import (
)
from litellm.utils import convert_to_model_response_object, declared_mid_conversation_system_support
from ..common_utils import OpenAIError
from ..common_utils import OpenAIError, without_cache_control
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
if TYPE_CHECKING:
@ -128,6 +128,29 @@ def _litellm_params_str(litellm_params: Mapping[str, object] | None, key: str) -
return value if isinstance(value, str) else None
def _drops_params(litellm_params: Mapping[str, object]) -> bool:
return drops_unsupported_params(litellm_params.get("drop_params"))
def _names_claude(model: str, litellm_params: Mapping[str, object]) -> bool:
base_model: Final = litellm_params.get("base_model")
return "claude" in model.lower() or (isinstance(base_model, str) and "claude" in base_model.lower())
def _claude_messages_within_cache_control_cap(
model: str,
messages: Sequence[AllMessageValues],
tools: Sequence["ChatCompletionToolParam"] | None,
litellm_params: Mapping[str, object],
request_params: Mapping[str, object],
) -> Sequence[AllMessageValues]:
if not _names_claude(model, litellm_params) or not _drops_params(litellm_params):
return messages
from litellm.integrations.anthropic_cache_control_hook import messages_within_cache_control_cap
return list(messages_within_cache_control_cap(messages, tools, request_params.get("cache_control"), request_params))
def _prompt_cache_options(optional_params: Mapping[str, object]) -> object | None:
try:
extra_body: Final = _OBJECT_MAPPING.validate_python(optional_params.get("extra_body"))
@ -311,52 +334,58 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
return content_copy
async def _async_handle_pdf_url(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
file_id: Final = content_item.get("file_id")
content_copy: Final = content_item.copy()
file_id: Final = content_copy.get("file_id")
if file_id is not None: # check for file id being url done in _handle_pdf_url
base64_data: Final = await async_convert_url_to_base64(file_id)
content_item["file_data"] = base64_data
content_item["filename"] = "my_file.pdf"
content_item.pop("file_id")
return content_item
content_copy["file_data"] = base64_data
content_copy["filename"] = "my_file.pdf"
content_copy.pop("file_id")
return content_copy
def _common_file_data_check(self, content_item: ChatCompletionFileObjectFile) -> ChatCompletionFileObjectFile:
file_data: Final = content_item.get("file_data")
filename: Final = content_item.get("filename")
if file_data is not None and filename is None:
content_item["filename"] = "my_file.pdf"
return {**content_item, "filename": "my_file.pdf"}
return content_item
@staticmethod
def _image_item_for_openai(content_item: ChatCompletionImageObject) -> ChatCompletionImageObject:
image_url: Final = content_item["image_url"]
if isinstance(image_url, str):
return {**content_item, "image_url": {"url": image_url}}
if not isinstance(image_url, dict):
return content_item
new_image_url_obj: Final = cast(
ChatCompletionImageUrlObject,
{k: v for k, v in image_url.items() if k not in {"format", "video_metadata"}},
)
return {**content_item, "image_url": new_image_url_obj}
@staticmethod
def _file_item_for_openai(content_item: ChatCompletionFileObject) -> ChatCompletionFileObject:
litellm_specific_params: Final = {"format", "detail", "video_metadata"}
file_obj: Final = content_item.get("file")
if file_obj is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=None,
llm_provider="openai",
)
new_file_obj: Final = ChatCompletionFileObjectFile(
**{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
)
return {**content_item, "file": new_file_obj}
def _apply_common_transform_content_item(
self,
content_item: OpenAIMessageContentListBlock,
) -> OpenAIMessageContentListBlock:
litellm_specific_params: Final = {"format", "detail", "video_metadata"}
if content_item.get("type") == "image_url":
content_item = cast(ChatCompletionImageObject, content_item)
if isinstance(content_item["image_url"], str):
content_item["image_url"] = {
"url": content_item["image_url"],
}
elif isinstance(content_item["image_url"], dict):
new_image_url_obj: Final = cast(
ChatCompletionImageUrlObject,
{k: v for k, v in content_item["image_url"].items() if k not in {"format", "video_metadata"}},
)
content_item["image_url"] = new_image_url_obj
elif content_item.get("type") == "file":
content_item = cast(ChatCompletionFileObject, content_item)
file_obj: Final = content_item.get("file")
if file_obj is None:
raise litellm.BadRequestError(
message="Content block has type='file' but is missing the required 'file' field",
model=None,
llm_provider="openai",
)
new_file_obj: Final = ChatCompletionFileObjectFile(
**{k: v for k, v in file_obj.items() if k not in litellm_specific_params}
)
content_item["file"] = new_file_obj
return self._image_item_for_openai(cast(ChatCompletionImageObject, content_item))
if content_item.get("type") == "file":
return self._file_item_for_openai(cast(ChatCompletionFileObject, content_item))
return content_item
def _transform_content_item(
@ -419,31 +448,34 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
hoisted_messages: Final = hoist_images_from_tool_messages(stripped_messages)
async def _async_transform():
for message in hoisted_messages:
message_content = message.get("content")
message_role = message.get("role")
if message_role == "user" and message_content and isinstance(message_content, list):
message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
for i, content_item in enumerate(message_content_types):
message_content_types[i] = await self._async_transform_content_item(
cast(OpenAIMessageContentListBlock, content_item),
)
return hoisted_messages
return [await self._async_user_message_for_openai(message) for message in hoisted_messages]
if is_async:
return _async_transform()
else:
for message in hoisted_messages:
message_content = message.get("content")
message_role = message.get("role")
if message_role == "user" and message_content and isinstance(message_content, list):
message_content_types = cast(list[OpenAIMessageContentListBlock], message_content)
for i, content_item in enumerate(message_content):
message_content_types[i] = self._transform_content_item(
cast(OpenAIMessageContentListBlock, content_item)
)
return hoisted_messages
return [self._user_message_for_openai(message) for message in hoisted_messages]
@staticmethod
def _user_content_parts(message: AllMessageValues) -> Sequence[OpenAIMessageContentListBlock] | None:
message_content: Final = message.get("content")
if message.get("role") != "user" or not message_content or not isinstance(message_content, list):
return None
return cast(Sequence[OpenAIMessageContentListBlock], message_content)
def _user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
parts: Final = self._user_content_parts(message)
if parts is None:
return message
return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI
AllMessageValues, {**message, "content": [self._transform_content_item(part) for part in parts]}
)
async def _async_user_message_for_openai(self, message: AllMessageValues) -> AllMessageValues:
parts: Final = self._user_content_parts(message)
if parts is None:
return message
return cast( # cast-ok: the same user message with each content part rebuilt for OpenAI
AllMessageValues, {**message, "content": [await self._async_transform_content_item(part) for part in parts]}
)
def transform_messages(
self,
@ -462,23 +494,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
messages: list[AllMessageValues],
tools: list["ChatCompletionToolParam"] | None = None,
) -> tuple[list[AllMessageValues], list["ChatCompletionToolParam"] | None]:
from litellm.litellm_core_utils.prompt_templates.common_utils import (
filter_value_from_dict,
)
from litellm.types.llms.openai import ChatCompletionToolParam
for i, message in enumerate(messages):
messages[i] = cast(
AllMessageValues,
filter_value_from_dict(message, "cache_control"),
)
if tools is not None:
for i, tool in enumerate(tools):
tools[i] = cast(
ChatCompletionToolParam,
filter_value_from_dict(tool, "cache_control"),
)
return messages, tools
new_messages: Final = [cast(AllMessageValues, without_cache_control(message)) for message in messages]
new_tools: Final = (
[cast(ChatCompletionToolParam, without_cache_control(tool)) for tool in tools]
if tools is not None
else None
)
return new_messages, new_tools
def translate_developer_role_to_system_role(
self,
@ -527,6 +551,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
tools: Sequence["ChatCompletionToolParam"] | None,
prompt_cache_options: object,
litellm_params: Mapping[str, object],
request_params: Mapping[str, object],
) -> tuple[Sequence[AllMessageValues], Sequence["ChatCompletionToolParam"] | None]:
"""
A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on
@ -543,10 +568,11 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
model, custom_llm_provider, api_base, prompt_cache_options
)
if not reads_breakpoints and self._should_preserve_cache_control_for_endpoint(custom_llm_provider, api_base):
return messages, tools
drops_breakpoints: Final = not reads_breakpoints and (
litellm.drop_params is True or litellm_params.get("drop_params") is True
)
return (
_claude_messages_within_cache_control_cap(model, messages, tools, litellm_params, request_params),
tools,
)
drops_breakpoints: Final = not reads_breakpoints and _drops_params(litellm_params)
renamed: Final = (
[AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
if reads_breakpoints
@ -561,7 +587,13 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
if drops_breakpoints
else list(tools)
)
return self.remove_cache_control_flag_from_messages_and_tools(model=model, messages=renamed, tools=kept_tools)
sent_messages, sent_tools = self.remove_cache_control_flag_from_messages_and_tools(
model=model, messages=renamed, tools=kept_tools
)
return (
_claude_messages_within_cache_control_cap(model, sent_messages, sent_tools, litellm_params, request_params),
sent_tools,
)
def _sanitized_tools_update_for_openai(
self,
@ -623,6 +655,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
tools=optional_params.get("tools", []),
prompt_cache_options=_prompt_cache_options(optional_params),
litellm_params=litellm_params,
request_params=optional_params,
)
if tools is not None and len(tools) > 0:
optional_params["tools"] = tools
@ -654,6 +687,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
tools=optional_params.get("tools", []),
prompt_cache_options=_prompt_cache_options(optional_params),
litellm_params=litellm_params,
request_params=optional_params,
)
if tools is not None and len(tools) > 0:
optional_params["tools"] = tools

View file

@ -153,12 +153,13 @@ class OpenAIOSeriesConfig(OpenAIGPTConfig):
- role: system ==> translate to role 'user'
"""
_supports_system_messages: Final = supports_system_messages(model, "openai")
for i, message in enumerate(messages):
transformed_messages: Final = list(messages)
for i, message in enumerate(transformed_messages):
if message["role"] == "system" and not _supports_system_messages:
new_message = ChatCompletionUserMessage(content=message["content"], role="user")
messages[i] = new_message # Replace the old message with the new one
transformed_messages[i] = new_message # Replace the old message with the new one
if is_async:
return super()._transform_messages(messages, model, is_async=cast(Literal[True], True))
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[True], True))
else:
return super()._transform_messages(messages, model, is_async=cast(Literal[False], False))
return super()._transform_messages(transformed_messages, model, is_async=cast(Literal[False], False))

View file

@ -3,6 +3,7 @@ Common helpers / utils across al OpenAI endpoints
"""
import asyncio
import copy
import hashlib
import inspect
import json
@ -31,7 +32,7 @@ if TYPE_CHECKING:
import litellm
from litellm.exceptions import error_response_without_credentials
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, iter_items
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning_params
from litellm.litellm_core_utils.token_counter import token_counter, usage_tokenizer_model
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -43,6 +44,47 @@ from litellm.llms.custom_httpx.http_handler import (
)
def _is_plain_dict(value: object) -> bool:
return isinstance(value, dict)
def _is_list(value: object) -> bool:
return isinstance(value, list)
def _contains_cache_control(value: object) -> bool:
pending: Final[list[object]] = [value] # mutable-ok: a local worklist, never returned or stored
while pending:
node = pending.pop()
mapping = as_str_mapping(node)
if mapping is not None:
if "cache_control" in mapping:
return True
pending.extend(mapping.values())
elif _is_list(node):
pending.extend(iter_items(node))
return False
def without_cache_control(item: object) -> object:
"""Return `item` with every `cache_control` key removed, without touching the original.
`filter_value_from_dict` deletes the key in place and recurses into nested dicts and
lists, so the caller's own message, tool or input object has to be copied first: it
may reuse the same list on a provider that does support prompt caching. The copy is
skipped when there is nothing to strip, which keeps the object identity the
pass-through paths rely on and avoids a deep copy on every request.
"""
from litellm.litellm_core_utils.prompt_templates.common_utils import filter_value_from_dict
if not _is_plain_dict(item) or not _contains_cache_control(item):
return item
copied: Final = copy.deepcopy(item)
if isinstance(copied, dict):
filter_value_from_dict(copied, "cache_control")
return copied
def _get_client_init_params(cls: type) -> tuple[str, ...]:
"""Extract __init__ parameter names (excluding 'self') from a class."""
return tuple(p for p in inspect.signature(cls.__init__).parameters if p != "self")

View file

@ -354,10 +354,10 @@ class OpenAIDecisionsConfig(BaseDecisionsConfig):
) -> str | None:
return resolve_openai_bearer_token(api_key=api_key, api_base=api_base, litellm_params=litellm_params)
def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool:
def supports_audio_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
return False
def supports_video_input(self, model: str, custom_llm_provider: str) -> bool:
def supports_video_input(self, model: str, custom_llm_provider: str, deployment_flag: bool | None) -> bool:
return False
def transform_decisions_request(

View file

@ -16,6 +16,7 @@ from openai._base_client import make_request_options
from openai._constants import RAW_RESPONSE_HEADER
from openai._legacy_response import LegacyAPIResponse
from openai._types import RequestOptions
from openai.pagination import SyncCursorPage
from openai.types import CreateEmbeddingResponse
from openai.types.beta.assistant_deleted import AssistantDeleted
from openai.types.chat import ChatCompletion, ChatCompletionChunk
@ -40,6 +41,7 @@ from litellm.types.utils import (
LiteLLMBatch,
ModelResponse,
ModelResponseStream,
openai_batch_status,
)
from litellm.utils import (
CustomStreamWrapper,
@ -2182,6 +2184,18 @@ class OpenAIFilesAPI(BaseLLM):
return response
def _with_openai_batch_fields(batches: Sequence[Batch]) -> Sequence[Batch]:
return [
batch.model_copy(
update={
"object": batch.object or "batch",
"status": openai_batch_status(batch.status),
}
)
for batch in batches
]
class OpenAIBatchesAPI(BaseLLM):
"""
OpenAI methods to support for batches
@ -2372,7 +2386,7 @@ class OpenAIBatchesAPI(BaseLLM):
):
verbose_logger.debug("listing batches, after= %s, limit= %s", after, limit)
response: Final = await openai_client.batches.list(after=after, limit=limit)
return response
return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
def list_batches(
self,
@ -2384,7 +2398,7 @@ class OpenAIBatchesAPI(BaseLLM):
organization: str | None,
after: str | None = None,
limit: int | None = None,
client: OpenAI | None = None,
client: OpenAI | AsyncOpenAI | None = None,
litellm_params: Mapping[str, object] | None = None,
):
openai_client: Final[OpenAI | AsyncOpenAI | None] = self.get_openai_client(
@ -2409,7 +2423,9 @@ class OpenAIBatchesAPI(BaseLLM):
)
return self.alist_batches(openai_client=openai_client, after=after, limit=limit)
response: Final = openai_client.batches.list(after=after, limit=limit)
return response
if not isinstance(response, SyncCursorPage):
return response
return response.model_copy(update={"data": _with_openai_batch_fields(response.data)})
class OpenAIAssistantsAPI(BaseLLM):

View file

@ -38,6 +38,7 @@ from itertools import accumulate, chain, groupby, repeat
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar, Union, cast
from openai.types.responses.response_output_refusal import ResponseOutputRefusal
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
@ -277,6 +278,7 @@ _TOOL_CALL_PAYLOAD_EVENT_TYPES: Final = _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES | f
)
_OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"})
_OUTPUT_TEXT_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.output_text.done"})
_TEXT_DELTA_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.refusal.delta"})
_TOOL_OUTPUT_TEXT_PART_TYPES: Final = frozenset({"input_text", "output_text", "text"})
_PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
{"function_call_output": "output", "custom_tool_call_output": "output", "message": "content"}
@ -537,6 +539,14 @@ def _scanned_text_field(item: Mapping[str, object]) -> str:
return "output" if item.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES else "content"
def _content_part_text_field(part: object) -> str:
return "refusal" if stream_item_field(part, "type") == "refusal" else "text"
def _has_writable_field(part: object, field: str) -> bool:
return isinstance(part, dict) or hasattr(part, field)
def _tool_output_part_text(part: object) -> str | None:
if not isinstance(part, dict):
return None
@ -993,7 +1003,7 @@ class OpenAIResponsesHandler(BaseTranslation):
for content_idx, content_item in enumerate(content):
if isinstance(content_item, dict):
# Extract text
text_str = content_item.get("text", None)
text_str = content_item.get(_content_part_text_field(content_item), None)
if text_str is not None:
texts_to_check.append(text_str)
task_mappings.append((msg_idx, int(content_idx)))
@ -1042,8 +1052,7 @@ class OpenAIResponsesHandler(BaseTranslation):
elif isinstance(content, list):
# Replace specific text item in list content
if isinstance(messages[msg_idx][field][content_idx_optional], dict):
messages[msg_idx][field][content_idx_optional]["text"] = guardrail_response
self._write_content_part_text(content[content_idx_optional], guardrail_response)
async def process_output_response(
self,
@ -1380,7 +1389,8 @@ class OpenAIResponsesHandler(BaseTranslation):
if isinstance(output_index, int) and isinstance(content_index, int)
)
if len(placeable_positions) != len(scanned_positions) or any(
stream_item_field(event, "type") not in _OUTPUT_TEXT_EVENT_TYPES for event in scanned_events
stream_item_field(event, "type") not in _OUTPUT_TEXT_EVENT_TYPES | _TEXT_DELTA_EVENT_TYPES
for event in scanned_events
):
from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
@ -1401,6 +1411,12 @@ class OpenAIResponsesHandler(BaseTranslation):
else:
setattr(event, field, value)
@staticmethod
def _write_content_part_text(part: object, text: str) -> None:
field: Final = _content_part_text_field(part)
if _has_writable_field(part, field):
OpenAIResponsesHandler._write_event_field(part, field, text)
@staticmethod
def _write_compaction_summary(item: object, summary: str) -> None:
encrypted_content: Final = stream_item_field(item, "encrypted_content")
@ -1416,11 +1432,12 @@ class OpenAIResponsesHandler(BaseTranslation):
) -> None:
"""Sync pre-completion stream events with the rewritten completed
response, keyed by ``(output_index, content_index)``: the first
``output_text.delta`` for a rewritten item carries the full rewritten
text and the rest are blanked, while ``output_text.done``,
``content_part.done``, and ``output_item.done`` events carry the full
rewritten text, so every event a client may read agrees with the
rewritten ``response.completed`` payload."""
``output_text.delta`` or ``refusal.delta`` for a rewritten item carries
the full rewritten text and the rest are blanked, while
``output_text.done``, ``refusal.done``, ``content_part.done``, and
``output_item.done`` events carry the full rewritten text, so every
event a client may read agrees with the rewritten ``response.completed``
payload."""
delta_replacements: Final = MappingProxyType(
{position: chain((rewritten,), repeat("")) for position, rewritten in rewrites_by_position.items()}
)
@ -1436,14 +1453,14 @@ class OpenAIResponsesHandler(BaseTranslation):
if not isinstance(output_index, int) or not isinstance(content_index, int):
continue
position = (output_index, content_index)
if event_type == "response.output_text.delta" and position in delta_replacements:
if event_type in _TEXT_DELTA_EVENT_TYPES and position in delta_replacements:
self._write_event_field(event, "delta", next(delta_replacements[position]))
elif event_type == "response.output_text.done" and position in rewrites_by_position:
self._write_event_field(event, "text", rewrites_by_position[position])
elif event_type == "response.refusal.done" and position in rewrites_by_position:
self._write_event_field(event, "refusal", rewrites_by_position[position])
elif event_type == "response.content_part.done" and position in rewrites_by_position:
part = event.get("part")
if isinstance(part, dict) or hasattr(part, "text"):
self._write_event_field(part, "text", rewrites_by_position[position])
self._write_content_part_text(event.get("part"), rewrites_by_position[position])
@staticmethod
def _sync_output_item_done_event(
@ -1459,7 +1476,7 @@ class OpenAIResponsesHandler(BaseTranslation):
for (item_idx, content_idx), rewritten in rewrites_by_position.items():
if item_idx != output_index or not 0 <= content_idx < len(content):
continue
OpenAIResponsesHandler._write_event_field(content[content_idx], "text", rewritten)
OpenAIResponsesHandler._write_content_part_text(content[content_idx], rewritten)
def _deliver_ended_stream_tool_call_rewrites(
self,
@ -1814,7 +1831,7 @@ class OpenAIResponsesHandler(BaseTranslation):
return
# Handle both GenericResponseOutputItem and dict
content: list[OutputText] | list[dict] | None = None
content: list[OutputText | ResponseOutputRefusal] | list[dict[str, object]] | None = None
if isinstance(output_item, BaseModel):
try:
output_item_dump: Final = output_item.model_dump()
@ -1837,17 +1854,9 @@ class OpenAIResponsesHandler(BaseTranslation):
verbose_proxy_logger.debug("OpenAI Responses API: Processing output item: %s", output_item)
# Iterate through content items (list of OutputText objects)
for content_idx, content_item in enumerate(content):
# Handle both OutputText objects and dicts
if isinstance(content_item, OutputText):
text_content = content_item.text
elif isinstance(content_item, dict):
text_content = content_item.get("text")
else:
continue
if text_content:
text_content = stream_item_field(content_item, _content_part_text_field(content_item))
if isinstance(text_content, str) and text_content:
texts_to_check.append(text_content)
task_mappings.append((output_idx, int(content_idx)))
@ -1887,33 +1896,24 @@ class OpenAIResponsesHandler(BaseTranslation):
# Handle both GenericResponseOutputItem, BaseModel, and dict
if isinstance(output_item, GenericResponseOutputItem):
if output_item.content and content_idx < len(output_item.content):
content_item = output_item.content[content_idx]
if isinstance(content_item, OutputText):
content_item.text = guardrail_response
elif isinstance(content_item, dict):
content_item["text"] = guardrail_response
self._write_content_part_text(output_item.content[content_idx], guardrail_response)
elif isinstance(output_item, BaseModel):
# Handle other Pydantic models by converting to GenericResponseOutputItem
try:
generic_item = GenericResponseOutputItem.model_validate(output_item.model_dump())
if generic_item.content and content_idx < len(generic_item.content):
content_item = generic_item.content[content_idx]
if isinstance(content_item, OutputText):
content_item.text = guardrail_response
# Update the original response output
if hasattr(output_item, "content") and output_item.content:
original_content = output_item.content[content_idx]
if hasattr(original_content, "text"):
original_content.text = guardrail_response
if (
generic_item.content
and content_idx < len(generic_item.content)
and hasattr(output_item, "content")
and output_item.content
):
self._write_content_part_text(output_item.content[content_idx], guardrail_response)
except Exception:
pass
elif isinstance(output_item, dict):
content = output_item.get("content", [])
if content and content_idx < len(content):
if isinstance(content[content_idx], dict):
content[content_idx]["text"] = guardrail_response
elif hasattr(content[content_idx], "text"):
content[content_idx].text = guardrail_response
self._write_content_part_text(content[content_idx], guardrail_response)
def build_block_sse_chunks(
self,

View file

@ -52,7 +52,7 @@ from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
from litellm.types.workload_identity import OPENAI_WIF_KWARGS_KEYS
from ..common_utils import OpenAIError
from ..common_utils import OpenAIError, without_cache_control
from ..workload_identity import get_workload_identity_bearer_token, resolve_openai_workload_identity_config
OPENAI_RESPONSES_API_MIN_MAX_OUTPUT_TOKENS: Final = 16
@ -658,13 +658,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
input=self._replayable_input(marker_safe_input, reads_breakpoints, maps_anthropic_tool_search),
tools=mapped_tools,
)
if marker_safe_input is not validated_input:
self._strip_caller_input_as_if_unmarked(
model=model,
caller_input=validated_input,
reads_breakpoints=reads_breakpoints,
maps_anthropic_tool_search=maps_anthropic_tool_search,
)
object_schema_tools: Final = self._tools_with_object_parameters(model=model, tools=stripped_tools)
sanitized_tools: Final = self._sanitized_tool_schemas_for_openai(
model=model, tools=object_schema_tools, litellm_params=litellm_params
@ -679,17 +672,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
return marked_input
return input_without_unanswered_anthropic_tool_search_calls(marked_input)
def _strip_caller_input_as_if_unmarked(
self,
model: str,
caller_input: str | ResponseInputParam,
reads_breakpoints: bool,
maps_anthropic_tool_search: bool,
) -> None:
self.remove_cache_control_flag_from_input_and_tools(
model=model, input=self._replayable_input(caller_input, reads_breakpoints, maps_anthropic_tool_search)
)
def _input_with_wire_safe_prompt_cache_breakpoints(
self,
model: str,
@ -800,24 +782,29 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
the chat path. Strips Anthropic-only `cache_control` markers from
Responses API input content blocks and tools.
`filter_value_from_dict` mutates each dict in place, so the same
objects are returned.
`filter_value_from_dict` deletes the key in place and recurses, so each
item is copied first: the caller keeps its own input list and may reuse
it on a provider that does support prompt caching.
"""
from litellm.litellm_core_utils.prompt_templates.common_utils import (
filter_value_from_dict,
new_input: Final = (
cast( # cast-ok: the comprehension rebuilds the input item for item
ResponseInputParam,
[without_cache_control(item) for item in input],
)
if isinstance(input, list)
else input
)
if isinstance(input, list):
for item in input:
if isinstance(item, dict):
filter_value_from_dict(cast(dict, item), "cache_control")
new_tools: Final = (
cast( # cast-ok: the comprehension rebuilds the tool list item for item
Sequence[ALL_RESPONSES_API_TOOL_PARAMS],
[without_cache_control(tool) for tool in tools],
)
if tools is not None
else None
)
if tools is not None:
for tool in tools:
if isinstance(tool, dict):
filter_value_from_dict(cast(dict, tool), "cache_control")
return input, tools
return new_input, new_tools
def _drop_bridge_minted_reasoning_items(self, input: str | ResponseInputParam) -> str | ResponseInputParam:
if self.custom_llm_provider not in _PROVIDERS_REPLAYING_ONLY_THEIR_OWN_REASONING or not isinstance(input, list):

View file

@ -125,6 +125,20 @@ The provider inherits all request/response handling from OpenAI's Responses API
If `supported_endpoints` is omitted, it defaults to `[]` (only chat completions, which is always enabled for JSON providers).
When only some models serve an endpoint natively, list them under `model_supported_endpoints`. Every other model reaches `/v1/responses` and `/v1/messages` through chat completions:
```json
{
"your_provider": {
"supported_endpoints": ["/v1/chat/completions"],
"model_supported_endpoints": {
"model-a": ["/v1/messages"],
"model-b": ["/v1/responses"]
}
}
}
```
### How It Works
1. `json_loader.py` checks `supported_endpoints` for `/v1/responses`

View file

@ -3,11 +3,16 @@
## Allows jina ai embedding calls - which don't allow 'encoding_format' in payload.
import json
import re
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final
import httpx
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@ -17,6 +22,57 @@ from litellm.types.utils import EmbeddingResponse
from ..common_utils import OpenAILikeBase, OpenAILikeError
_KEY_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str])
_EXTRA_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_TRANSPORT_HEADER_NAMES: Final = frozenset(
{
"host",
"content-length",
"transfer-encoding",
"connection",
"keep-alive",
"proxy-connection",
"te",
"trailer",
"upgrade",
"expect",
}
)
_HEADER_NAME: Final = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
_HEADER_VALUE: Final = re.compile(r"[\t\x20-\x7e]*")
def _header_text(name: str, value: object) -> str | None:
if isinstance(value, bool) or not isinstance(value, str | int | float):
return None
if _HEADER_NAME.fullmatch(name) is None:
return None
return str(value).strip(" \t\r\n")
def _extra_headers_as_text(extra_headers: Mapping[str, object], model: str, drop_params: bool) -> Mapping[str, str]:
caller_set: Final = {
name: value for name, value in extra_headers.items() if name.lower() not in _TRANSPORT_HEADER_NAMES
}
texts: Final = MappingProxyType({name: _header_text(name, value) for name, value in caller_set.items()})
refused: Final = sorted(name for name, text in texts.items() if text is None)
if refused and not drop_params:
raise litellm.UnsupportedParamsError(
message=f"extra_headers {', '.join(map(repr, refused))} cannot be sent. A header needs a name made of "
"letters, digits or !#$%&'*+-.^_`|~ and a text or number value. "
"To drop them from the call, set `litellm.drop_params = True`.",
model=model,
)
named: Final = MappingProxyType({name: text for name, text in texts.items() if text is not None})
unsendable: Final = sorted(name for name, text in named.items() if _HEADER_VALUE.fullmatch(text) is None)
if unsendable:
verbose_logger.warning(
"extra_headers %s dropped for model %r, since a header value must be printable ASCII on one line",
unsendable,
model,
)
return MappingProxyType({name: text for name, text in named.items() if name not in unsendable})
class OpenAILikeEmbeddingHandler(OpenAILikeBase):
def __init__(self, **kwargs):
@ -94,6 +150,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
aembedding=None,
custom_endpoint: bool | None = None,
headers: dict | None = None,
drop_params: bool = False,
) -> EmbeddingResponse:
api_base, headers = self._validate_environment(
api_base=api_base,
@ -103,8 +160,20 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
custom_endpoint=custom_endpoint,
)
model = model
filtered_optional_params: Final = {k: v for k, v in optional_params.items() if v not in (None, "")}
data: Final = {"model": model, "input": input, **filtered_optional_params}
data: Final = {
"model": model,
"input": input,
**{k: v for k, v in optional_params.items() if k != "extra_headers" and v not in (None, "")},
}
extra_headers: Final = _extra_headers_as_text(
_EXTRA_HEADERS_ADAPTER.validate_python(optional_params.get("extra_headers") or {}), model, drop_params
)
key_headers: Final = _KEY_HEADERS_ADAPTER.validate_python(headers)
extra_header_names: Final = frozenset(name.lower() for name in extra_headers)
request_headers: Final = {
**{name: value for name, value in key_headers.items() if name.lower() not in extra_header_names},
**extra_headers,
}
## LOGGING
logging_obj.pre_call(
@ -123,7 +192,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
api_key=api_key,
timeout=timeout,
client=client,
headers=headers,
headers=request_headers,
)
if client is None or isinstance(client, AsyncHTTPHandler):
self.client = HTTPHandler(timeout=timeout)
@ -134,7 +203,7 @@ class OpenAILikeEmbeddingHandler(OpenAILikeBase):
try:
response: Final = self.client.post(
api_base,
headers=headers,
headers=request_headers,
data=json.dumps(data),
)

View file

@ -3,7 +3,7 @@ JSON-based provider configuration loader for OpenAI-compatible providers.
"""
import json
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from pathlib import Path
from typing import Final
@ -14,6 +14,7 @@ from litellm.types.utils import ProviderSpecificModelInfo
_PROVIDER_INFO_ADAPTER: Final = TypeAdapter(ProviderSpecificModelInfo)
_SPECIAL_HANDLING_ADAPTER: Final = TypeAdapter(Mapping[str, bool])
_MODEL_ENDPOINTS_ADAPTER: Final = TypeAdapter(Mapping[str, tuple[str, ...]])
class SimpleProviderConfig:
@ -30,7 +31,10 @@ class SimpleProviderConfig:
self.special_handling: Mapping[str, bool] = _SPECIAL_HANDLING_ADAPTER.validate_python(
data.get("special_handling", {})
)
self.supported_endpoints = data.get("supported_endpoints", [])
self.supported_endpoints: Sequence[str] = data.get("supported_endpoints", [])
self.model_supported_endpoints: Mapping[str, tuple[str, ...]] = _MODEL_ENDPOINTS_ADAPTER.validate_python(
data.get("model_supported_endpoints", {})
)
self.provider_info: ProviderSpecificModelInfo = (
_PROVIDER_INFO_ADAPTER.validate_python(data["provider_info"]) if "provider_info" in data else {}
)
@ -90,6 +94,23 @@ class JSONProviderRegistry:
return False
return "/v1/responses" in provider.supported_endpoints
@classmethod
def supports_embeddings(cls, slug: str) -> bool:
provider: Final = cls._providers.get(slug)
if provider is None:
return False
return "/v1/embeddings" in provider.supported_endpoints
@classmethod
def serves_endpoint(cls, slug: str, model: str | None, endpoint: str) -> bool:
provider: Final = cls._providers.get(slug)
if provider is None:
return False
model_endpoints: Final = (
provider.model_supported_endpoints.get(model.removeprefix(f"{slug}/"), ()) if model else ()
)
return endpoint in provider.supported_endpoints or endpoint in model_endpoints
@classmethod
def list_providers(cls) -> list:
"""List all registered provider slugs"""

View file

@ -211,7 +211,7 @@
"param_mappings": {
"max_completion_tokens": "max_tokens"
},
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"]
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"]
},
"scx-ai": {
"base_url": "https://api.scx.ai/v1",
@ -231,6 +231,16 @@
"api_base_env": "PRISM_API_BASE",
"supported_endpoints": ["/v1/chat/completions", "/v1/responses", "/v1/messages"]
},
"hostinger_router": {
"base_url": "https://router.hostinger.com/v1",
"api_key_env": "HOSTINGER_ROUTER_API_KEY",
"api_base_env": "HOSTINGER_ROUTER_API_BASE",
"supported_endpoints": ["/v1/chat/completions", "/v1/embeddings"],
"model_supported_endpoints": {
"claude-sonnet-5": ["/v1/messages"],
"gpt-5.2": ["/v1/responses"]
}
},
"sail": {
"base_url": "https://api.sailresearch.com/v1",
"api_key_env": "SAIL_API_KEY",

View file

@ -110,38 +110,23 @@ def separate_cached_messages(
messages: list[AllMessageValues],
) -> tuple[list[AllMessageValues], list[AllMessageValues]]:
"""
Returns separated cached and non-cached messages.
Args:
messages: List of messages to be separated.
Returns:
Tuple containing:
- cached_messages: List of cached messages.
- non_cached_messages: List of non-cached messages.
Returns the first continuous block of cached messages, plus every system message once
any message is cached, since generateContent rejects a system_instruction beside a
cachedContent, and the remaining messages.
"""
cached_messages: list[AllMessageValues] = []
non_cached_messages: list[AllMessageValues] = []
filtered_messages: Final = [(idx, message) for idx, message in enumerate(messages) if is_cached_message(message)]
if not filtered_messages:
return [], messages
# Extract cached messages and their indices
filtered_messages: Final[list[tuple[int, AllMessageValues]]] = []
for idx, message in enumerate(messages):
if is_cached_message(message=message):
filtered_messages.append((idx, message))
# Validate only one block of continuous cached messages
last_continuous_block_idx: Final = get_first_continuous_block_idx(filtered_messages)
# Separate messages based on the block of cached messages
if filtered_messages and last_continuous_block_idx is not None:
first_cached_idx: Final = filtered_messages[0][0]
last_cached_idx: Final = filtered_messages[last_continuous_block_idx][0]
cached_messages = messages[first_cached_idx : last_cached_idx + 1]
non_cached_messages = messages[:first_cached_idx] + messages[last_cached_idx + 1 :]
else:
non_cached_messages = messages
return cached_messages, non_cached_messages
cached_block: Final = range(filtered_messages[0][0], filtered_messages[last_continuous_block_idx][0] + 1)
goes_in_cache: Final = tuple(
idx in cached_block or message.get("role") == "system" for idx, message in enumerate(messages)
)
return (
[message for message, cached in zip(messages, goes_in_cache) if cached],
[message for message, cached in zip(messages, goes_in_cache) if not cached],
)
def cached_messages_end_on_supported_turn(cached_messages: Sequence[AllMessageValues]) -> bool:

View file

@ -31,9 +31,8 @@ from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
from litellm.litellm_core_utils.prompt_templates.factory import (
encode_tool_call_id_with_signature,
)
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
from litellm.litellm_core_utils.prompt_templates.factory import encode_tool_call_id_with_signature
from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
from litellm.llms.custom_httpx.http_handler import (
@ -2091,6 +2090,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
parts: list[HttpxPartType],
cumulative_tool_call_idx: int,
is_function_call: bool | None,
thought_signature_in_tool_call_id: bool = True,
) -> tuple[
ChatCompletionToolCallFunctionChunk | None,
list[ChatCompletionToolCallChunk] | None,
@ -2129,9 +2129,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
# back on the matching `functionResponse`.
if gemini_call_id:
_tool_response_chunk["id"] = gemini_call_id
# Embed thought signature in ID for OpenAI client compatibility
if thought_signature:
_tool_response_chunk["provider_specific_fields"] = {"thought_signature": thought_signature}
if thought_signature and thought_signature_in_tool_call_id:
_tool_response_chunk["id"] = encode_tool_call_id_with_signature(
_tool_response_chunk["id"] or "", thought_signature
)
@ -2793,6 +2793,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
model_response: Union[ModelResponse, "ModelResponseStream"],
standard_optional_params: dict,
cumulative_tool_call_index: int = 0,
thought_signature_in_tool_call_id: bool = True,
) -> tuple[list[dict], list[dict], list, list, int]:
"""
Helper method to process candidates and extract metadata
@ -2895,6 +2896,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
parts=candidate["content"]["parts"],
cumulative_tool_call_idx=cumulative_tool_call_index,
is_function_call=is_function_call(standard_optional_params),
thought_signature_in_tool_call_id=thought_signature_in_tool_call_id,
)
if "logprobsResult" in candidate:
@ -2978,6 +2980,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
model_response: Union[ModelResponse, "ModelResponseStream"],
standard_optional_params: dict[str, object], # mutable-ok: mirrors override contract
cumulative_tool_call_index: int = 0,
thought_signature_in_tool_call_id: bool = True,
) -> tuple[ # mutable-ok: mirrors override contract
list[dict[str, object]],
list[dict[str, object]],
@ -2986,7 +2989,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
int,
]:
return cls._process_candidates(
_candidates, model_response, standard_optional_params, cumulative_tool_call_index
_candidates,
model_response,
standard_optional_params,
cumulative_tool_call_index,
thought_signature_in_tool_call_id,
)
def transform_response(
@ -3069,7 +3076,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
safety_ratings,
citation_metadata,
_, # cumulative_tool_call_index not needed in non-streaming
) = VertexGeminiConfig._process_candidates(_candidates, model_response, logging_obj.optional_params)
) = VertexGeminiConfig._process_candidates(
_candidates,
model_response,
logging_obj.optional_params,
thought_signature_in_tool_call_id=embeds_thought_signature_in_tool_call_id(
logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped
),
)
usage: Final = VertexGeminiConfig._calculate_usage(
completion_response=completion_response,
@ -3905,6 +3919,9 @@ class ModelResponseIterator:
self.logging_obj = logging_obj
self.response_headers = response_headers or {}
self.is_function_call = check_is_function_call(logging_obj)
self.thought_signature_in_tool_call_id: Final = embeds_thought_signature_in_tool_call_id(
logging_obj.litellm_params # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging.litellm_params is untyped
)
self.cumulative_tool_call_index: int = 0
self.has_seen_tool_calls: bool = False
self._resume = resume
@ -3969,6 +3986,7 @@ class ModelResponseIterator:
model_response,
self.logging_obj.optional_params,
cumulative_tool_call_index=self.cumulative_tool_call_index,
thought_signature_in_tool_call_id=self.thought_signature_in_tool_call_id,
)
# Track whether tool_calls have been seen across streaming chunks.

View file

@ -4,6 +4,7 @@ Translation from OpenAI's `/chat/completions` endpoint to IBM WatsonX's `/text/c
Docs: https://cloud.ibm.com/apidocs/watsonx-ai#text-chat
"""
import copy
from typing import Final
from litellm import verbose_logger
@ -57,7 +58,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig):
_tools = non_default_params.pop("tools", None)
if _tools is not None:
# remove 'additionalProperties' from tools
_tools = remove_additional_properties(_tools)
_tools = remove_additional_properties(copy.deepcopy(_tools)) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
# remove 'strict' from tools
_tools = remove_strict_from_schema(_tools)
if _tools is not None:

View file

@ -1,3 +1,4 @@
import copy
from collections.abc import AsyncIterator, Iterator, Mapping
from types import MappingProxyType
from typing import Any, Final
@ -208,7 +209,8 @@ class XAIChatConfig(OpenAIGPTConfig):
tools = [
filtered_tool
for filtered_tool in (
_FILTERED_TOOL.validate_python(filter_value_from_dict(tool, "strict")) for tool in value
_FILTERED_TOOL.validate_python(filter_value_from_dict(copy.deepcopy(tool), "strict")) # pyright: ignore[reportUnknownArgumentType] # tools arrive untyped through the bare-dict params
for tool in value
)
if filtered_tool is not None
]

View file

@ -23,6 +23,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Mappin
from concurrent import futures
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
from copy import deepcopy
from dataclasses import dataclass
from functools import partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args
@ -82,8 +83,8 @@ from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.get_litellm_params import (
AWS_CREDENTIAL_KWARGS_KEYS,
OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS,
OPTIONAL_KWARGS_KEYS,
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
SUPPLEMENTAL_KWARGS_KEYS,
InvalidControlOption,
parse_control_options,
with_control_options,
@ -157,7 +158,7 @@ from litellm.types.completion import (
)
from litellm.types.litellm_params import ControlOptions, RetryStrategy
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import GenericLiteLLMParams
from litellm.types.router import GenericLiteLLMParams, validate_litellm_params_strictly
from litellm.types.utils import (
AgenticLoopParams,
BaseUrlRoutingHint,
@ -399,6 +400,49 @@ def _resolve_control_options(kwargs: Mapping[str, object], model: str) -> Contro
return assert_never(control)
@dataclass(frozen=True, slots=True)
class _SupplementalProviderParams:
params: GenericLiteLLMParams
dropped_keys: frozenset[str] = frozenset()
@dataclass(frozen=True, slots=True)
class _InvalidSupplementalParams:
message: str
param: str
def _parse_supplemental_provider_params(
kwargs: Mapping[str, object],
) -> _SupplementalProviderParams | _InvalidSupplementalParams:
supplemental: Final = {key: kwargs[key] for key in SUPPLEMENTAL_KWARGS_KEYS if key in kwargs}
try:
return _SupplementalProviderParams(params=validate_litellm_params_strictly(GenericLiteLLMParams, supplemental))
except ValidationError as error:
invalid: Final = tuple(detail for detail in error.errors() if detail["loc"])
invalid_keys: Final = frozenset(str(detail["loc"][0]) for detail in invalid)
if litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True:
return _SupplementalProviderParams(
params=GenericLiteLLMParams.model_validate(
{key: value for key, value in supplemental.items() if key not in invalid_keys}
),
dropped_keys=invalid_keys,
)
return _InvalidSupplementalParams(
message="; ".join(f"{detail['loc'][0]}: {detail['msg']}" for detail in invalid),
param=", ".join(sorted(invalid_keys)),
)
def _supplemental_provider_params(kwargs: Mapping[str, object], model: str) -> _SupplementalProviderParams:
parsed: Final = _parse_supplemental_provider_params(kwargs)
if isinstance(parsed, _InvalidSupplementalParams):
raise litellm.BadRequestError(
message=parsed.message, model=model, llm_provider=None, body={"param": parsed.param}
)
return parsed
PROVIDERS_WITHOUT_DOCUMENT_INPUT: Final = frozenset({"ollama", "ollama_chat"})
PROVIDERS_DROPPING_UNDECLARED_PDF_INPUT: Final = frozenset({"azure_ai"})
@ -830,21 +874,17 @@ async def acompletion(
"enable_json_schema_validation": enable_json_schema_validation,
}
if custom_llm_provider is None:
_supplemental: Final = cast( # cast-ok: kwargs values are request-scoped objects
"dict[str, object]",
{
**{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs},
"extra_headers": extra_headers,
},
)
request_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: untyped request kwargs
_, custom_llm_provider, _, _ = get_llm_provider(
model=model,
custom_llm_provider=custom_llm_provider,
api_base=cast( # cast-ok: api_base arrives as a str in the request kwargs
"str | None", kwargs.get("api_base")
"str | None", request_kwargs.get("api_base")
)
or base_url,
litellm_params=(GenericLiteLLMParams.model_validate(_supplemental) if _supplemental else None),
litellm_params=_supplemental_provider_params(
_REQUEST_KWARGS_ADAPTER.validate_python({**kwargs, "extra_headers": extra_headers}), model
).params,
)
if fallbacks is not None:
@ -4382,6 +4422,7 @@ _ADDITIONAL_DROP_PARAMS_ADAPTER: Final = TypeAdapter(list[str])
_OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object])
_OLLAMA_EMBEDDING_HEADERS_ADAPTER: Final = TypeAdapter(dict[str, str])
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints)
_BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint)
_OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object)
@ -5625,6 +5666,11 @@ def completion(
######### unpacking kwargs #####################
args: Final = _locals_snapshot(locals())
supplemental_params: Final = _supplemental_provider_params(
_REQUEST_KWARGS_ADAPTER.validate_python({**kwargs, "extra_headers": extra_headers}), model
)
for dropped_key in supplemental_params.dropped_keys:
kwargs.pop(dropped_key)
# Set by the responses->completion fallback so completion() does not bridge
# back to the Responses API: that round-trip mutually recurses forever for a
@ -5850,21 +5896,14 @@ def completion(
if deployment_id is not None: # azure llms
model = deployment_id
custom_llm_provider = "azure"
_supplemental_provider_params: Final = {
**{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs},
"extra_headers": extra_headers,
}
routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict
requested_model: Final = model
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
model=model,
custom_llm_provider=custom_llm_provider,
api_base=api_base,
api_key=api_key,
litellm_params=(
GenericLiteLLMParams.model_validate(_supplemental_provider_params)
if _supplemental_provider_params
else None
),
litellm_params=supplemental_params.params,
)
_record_agentic_loop_params(
logging_obj=logging,
@ -5884,7 +5923,6 @@ def completion(
api_base=api_base,
deployment_model_info=deployment_model_info,
)
routing_kwargs: Final = cast("Mapping[str, object]", kwargs) # cast-ok: kwargs is the untyped request dict
drop_unsupported_reasoning: Final = (
litellm.drop_params is True or normalize_drop_params(routing_kwargs.get("drop_params")) is True
)
@ -6220,6 +6258,7 @@ def completion(
"github_copilot_user_session",
PROVIDER_AFFINITY_HEADER_KWARG_KEY,
"fireworks_forward_user_id",
"thought_signature_in_tool_call_id",
"responses_api_version",
"allowed_openai_params",
)
@ -6809,6 +6848,23 @@ async def aembedding(*args, **kwargs) -> EmbeddingResponse:
)
def _embedding_headers(param: str, configured: object, model: str, drop_params: bool) -> Mapping[str, object]:
if not configured:
return {}
if isinstance(configured, Mapping):
return dict(cast(Mapping[str, object], configured)) # cast-ok: header names from untyped request JSON
try:
return dict(cast(Iterable[tuple[str, object]], configured)) # cast-ok: name and value pairs from request JSON
except (TypeError, ValueError):
if drop_params:
return {}
raise litellm.UnsupportedParamsError(
message=f"{param} must be a JSON object of header names to values. "
"To drop it from the call, set `litellm.drop_params = True`.",
model=model,
)
# fmt: off
# Overload for when aembedding=True (returns coroutine)
@ -6923,11 +6979,10 @@ def embedding(
azure_ad_token_provider: Final = kwargs.get("azure_ad_token_provider", None)
aembedding: Final[bool | None] = kwargs.get("aembedding", None)
extra_headers: Final = kwargs.get("extra_headers", None)
headers = kwargs.get("headers", None) or extra_headers
if headers is None:
headers = {}
if extra_headers is not None:
headers.update(extra_headers)
headers = {
**_embedding_headers("headers", kwargs.get("headers"), model, drop_unsupported_params),
**_embedding_headers("extra_headers", extra_headers, model, drop_unsupported_params),
}
# Inject proxy auth headers if configured
if litellm.proxy_auth is not None:
try:
@ -7091,7 +7146,7 @@ def embedding(
# set API KEY
api_key = api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY")
if headers is not None and headers != {}:
if headers:
optional_params["extra_headers"] = headers
requested_encoding_format: Final = (
@ -7183,14 +7238,15 @@ def embedding(
custom_llm_provider == "openai_like"
or custom_llm_provider == "llamafile"
or custom_llm_provider == "lm_studio"
or JSONProviderRegistry.supports_embeddings(custom_llm_provider)
):
api_base = api_base or litellm.api_base or get_secret_str("OPENAI_LIKE_API_BASE")
# set API KEY
if api_key is None:
api_key = api_key or litellm.api_key or litellm.openai_like_key or get_secret_str("OPENAI_LIKE_API_KEY")
if api_key is None and not JSONProviderRegistry.supports_embeddings(custom_llm_provider):
api_key = litellm.api_key or litellm.openai_like_key or get_secret_str("OPENAI_LIKE_API_KEY")
if headers is not None and headers != {}:
if headers:
optional_params["extra_headers"] = headers
## EMBEDDING CALL
@ -7205,10 +7261,9 @@ def embedding(
optional_params=optional_params,
client=client,
aembedding=aembedding,
drop_params=drop_unsupported_params,
)
elif custom_llm_provider == "oci":
if headers is None:
headers = {}
response = base_llm_http_handler.embedding(
model=model,
input=input,
@ -7233,11 +7288,6 @@ def embedding(
or litellm.api_key
)
# Use the merged headers variable (already merged at the top of the function)
# Don't overwrite it with just extra_headers
if headers is None:
headers = {}
response = base_llm_http_handler.embedding(
model=model,
input=input,

View file

@ -39687,6 +39687,57 @@
"/v1/images/generations"
]
},
"hostinger_router/gpt-5.2": {
"cache_read_input_token_cost": 0.00000021,
"input_cost_per_token": 0.0000021,
"litellm_provider": "hostinger_router",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 0.0000168,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_vision": true,
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/claude-sonnet-5": {
"cache_creation_input_token_cost": 3.3e-06,
"cache_creation_input_token_cost_above_1hr": 5.28e-06,
"cache_read_input_token_cost": 2.64e-07,
"input_cost_per_token": 2.64e-06,
"litellm_provider": "hostinger_router",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.32e-05,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_response_schema": true,
"supported_endpoints": ["/v1/chat/completions", "/v1/messages"],
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/text-embedding-3-large": {
"input_cost_per_token": 1.716e-07,
"litellm_provider": "hostinger_router",
"max_input_tokens": 8191,
"mode": "embedding",
"output_cost_per_token": 0.0,
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/text-embedding-3-small": {
"input_cost_per_token": 3e-08,
"litellm_provider": "hostinger_router",
"max_input_tokens": 8191,
"mode": "embedding",
"output_cost_per_token": 0.0,
"source": "https://router.hostinger.com/v1/models"
},
"hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": {
"input_cost_per_token": 1.2e-07,
"litellm_provider": "hyperbolic",
@ -51387,6 +51438,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 5e-06,
"output_cost_per_token_batches": 2.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -51415,6 +51471,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 5e-06,
"output_cost_per_token_batches": 2.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -52161,6 +52222,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_batches": 7.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -52266,6 +52332,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_batches": 7.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,

View file

@ -1087,6 +1087,32 @@
"generateContent": false
}
},
"hostinger_router": {
"display_name": "Hostinger Router (`hostinger_router`)",
"url": "https://docs.litellm.ai/docs/providers/hostinger_router",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": true,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
},
"crusoe": {
"display_name": "Crusoe (`crusoe`)",
"url": "https://docs.litellm.ai/docs/providers/crusoe",

View file

@ -24268,7 +24268,14 @@
"properties": {
"content": {
"items": {
"$ref": "#/components/schemas/OutputText"
"anyOf": [
{
"$ref": "#/components/schemas/OutputText"
},
{
"$ref": "#/components/schemas/ResponseOutputRefusal"
}
]
},
"title": "Content",
"type": "array"
@ -58015,7 +58022,7 @@
"paths": {
"/toolset/{toolset_name}/mcp": {
"delete": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_delete",
"parameters": [
{
@ -58054,7 +58061,7 @@
]
},
"get": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_get",
"parameters": [
{
@ -58093,7 +58100,7 @@
]
},
"head": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_head",
"parameters": [
{
@ -58132,7 +58139,7 @@
]
},
"options": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_options",
"parameters": [
{
@ -58171,7 +58178,7 @@
]
},
"patch": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_patch",
"parameters": [
{
@ -58210,7 +58217,7 @@
]
},
"post": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_post",
"parameters": [
{
@ -58249,7 +58256,7 @@
]
},
"put": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: API keys other than an admin's key with no\nteam must hold the toolset through one of their grant sources, or the\nrequest will be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put",
"parameters": [
{

View file

@ -5031,8 +5031,9 @@ class ProxyBaseLLMRequestProcessing:
if stream_usage is None:
return None
service_tier: Final = obj.get("service_tier")
served_model: Final = litellm_logging_obj.model if litellm_logging_obj is not None else None
cost_val: Final = ProxyBaseLLMRequestProcessing._streamed_usage_cost(
ModelResponse(usage=stream_usage),
ModelResponse(model=served_model, usage=stream_usage),
model_name,
service_tier if isinstance(service_tier, str) else None,
litellm_logging_obj,

View file

@ -12,18 +12,31 @@ from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.llms.anthropic.messages.native.utils import filtered_safeguard_results, redacted_stop_details
from litellm.llms.anthropic.messages.stream_assembly import anthropic_message_delta_fields
from litellm.llms.anthropic.messages.stream_assembly import (
anthropic_message_delta_fields,
anthropic_sse_error_events,
anthropic_sse_event_type,
anthropic_sse_event_types,
anthropic_sse_started_blocks,
anthropic_sse_stopped_block_index,
anthropic_sse_unfinished_blocks,
)
from litellm.llms.anthropic.messages.stream_assembly import (
assemble_anthropic_sse_stream as assemble_anthropic_sse_stream, # noqa: PLC0414 # public re-export
)
from litellm.llms.anthropic.messages.stream_assembly import (
has_unreadable_json_event as has_unreadable_json_event, # noqa: PLC0414 # public re-export
)
from litellm.llms.anthropic.messages.stream_assembly import (
is_anthropic_sse_stream as is_anthropic_sse_stream, # noqa: PLC0414 # public re-export
)
from litellm.llms.anthropic.messages.stream_assembly import (
is_raw_sse_stream as is_raw_sse_stream, # noqa: PLC0414 # public re-export
)
from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse
from litellm.types.utils import Choices, ModelResponse
@ -101,6 +114,48 @@ def _rewritten_message_delta_fields(
)
_MESSAGE_END_EVENT_TYPES: Final = frozenset({"message_delta", "message_stop"})
def _text_block_left_open(rewritten_chunks: Sequence[bytes], source_chunks: Sequence[object]) -> int | None:
"""Index of the rewritten text block to leave open when the failed upstream never stopped its text.
Assembly keeps a cut-off text block and drops every other cut-off block, and the round trip
merges all text into one block, so that block carries text the upstream left open. It stays
open only when it is the last block, since a client must see each block stop before the next
one starts.
"""
upstream_left_text_open: Final = any(
block.get("type") == "text" for block in anthropic_sse_unfinished_blocks(source_chunks).values()
)
rewritten_blocks: Final = anthropic_sse_started_blocks(rewritten_chunks)
if not upstream_left_text_open or not rewritten_blocks:
return None
last_index: Final = max(rewritten_blocks)
return last_index if rewritten_blocks[last_index].get("type") == "text" else None
def _is_empty_text(block: object) -> bool:
fields: Final = as_str_mapping(block)
return fields is not None and fields.get("type") == "text" and not fields.get("text")
def _without_empty_text(response: AnthropicMessagesResponse) -> AnthropicMessagesResponse:
return {**response, "content": [block for block in response.get("content") or () if not _is_empty_text(block)]}
def _sent_by_the_failed_upstream(chunk: bytes, block_left_open: int | None, end_events_sent: frozenset[str]) -> bool:
event: Final = chunk.decode()
event_type: Final = anthropic_sse_event_type(event)
if event_type in _MESSAGE_END_EVENT_TYPES:
return event_type in end_events_sent
return block_left_open is None or anthropic_sse_stopped_block_index(event) != block_left_open
def _sse_chunks_of_events(events: Sequence[str]) -> tuple[bytes, ...]:
return tuple(f"{event}\n\n".encode() for event in events)
def anthropic_sse_chunks_from_response(
assembled: ModelResponse, source_chunks: Sequence[object] = (), *, keep_explanations: bool = False
) -> tuple[bytes, ...]:
@ -110,7 +165,26 @@ def anthropic_sse_chunks_from_response(
nothing the OpenAI round trip made up, with a verdict kept only for a tool call delivered with
the input the classifier judged. Their free-text ``explanation`` was written about the text the
guardrail rewrote, so it is dropped unless the caller rewrites no text (``keep_explanations``).
A source stream that ended in an upstream ``error`` event ends with that event again, never
with a ``message_delta`` or ``message_stop`` the upstream did not send, with its cut-off text
block left open unless another block follows it, and with no finished text block when it read
no text, since Claude Code does not retry a stream that shows a finished block. A response that
cannot be re-emitted, such as one with a tool input that cannot be parsed, leaves a failed stream
its error alone.
"""
error_events: Final = anthropic_sse_error_events(source_chunks)
try:
return _rebuilt_sse_chunks(assembled, source_chunks, error_events, keep_explanations)
except Exception:
if not error_events:
raise
verbose_proxy_logger.warning("Failed Anthropic stream could not be rebuilt, sending its upstream error alone")
return _sse_chunks_of_events(error_events)
def _rebuilt_sse_chunks(
assembled: ModelResponse, source_chunks: Sequence[object], error_events: Sequence[str], keep_explanations: bool
) -> tuple[bytes, ...]:
from litellm.llms.anthropic.messages.adapters.transformation import (
LiteLLMAnthropicMessagesAdapter,
)
@ -120,7 +194,7 @@ def anthropic_sse_chunks_from_response(
adapter: Final = LiteLLMAnthropicMessagesAdapter()
anthropic_response: Final = adapter.translate_openai_response_to_anthropic(response=assembled)
source_response: Final = assemble_anthropic_sse_stream(source_chunks)
source_response: Final = assemble_anthropic_sse_stream(source_chunks, through_upstream_error=True)
source_content: Final = (
adapter.translate_openai_response_to_anthropic(response=source_response).get("content")
if source_response is not None
@ -129,11 +203,15 @@ def anthropic_sse_chunks_from_response(
unchanged_tool_use_ids: Final = _unchanged_tool_use_ids(
source_content or (), anthropic_response.get("content") or ()
)
return tuple(
FakeAnthropicMessagesStreamIterator(
response=anthropic_response,
message_delta_fields=_rewritten_message_delta_fields(
source_chunks, unchanged_tool_use_ids, keep_explanations
),
).chunks
rewritten_chunks: Final = FakeAnthropicMessagesStreamIterator(
response=_without_empty_text(anthropic_response) if error_events else anthropic_response,
message_delta_fields=_rewritten_message_delta_fields(source_chunks, unchanged_tool_use_ids, keep_explanations),
).chunks
if not error_events:
return tuple(rewritten_chunks)
block_left_open: Final = _text_block_left_open(rewritten_chunks, source_chunks)
end_events_sent: Final = _MESSAGE_END_EVENT_TYPES & anthropic_sse_event_types(source_chunks)
return (
*(chunk for chunk in rewritten_chunks if _sent_by_the_failed_upstream(chunk, block_left_open, end_events_sent)),
*_sse_chunks_of_events(error_events),
)

View file

@ -48,6 +48,7 @@ from litellm.proxy.common_utils.sse_keepalive import split_complete_sse_frames
from litellm.proxy.guardrails.anthropic_sse import (
anthropic_sse_chunks_from_response,
assemble_anthropic_sse_stream,
has_unreadable_json_event,
is_anthropic_sse_stream,
model_response_text,
)
@ -117,7 +118,7 @@ _MAX_FIRST_SSE_FRAME_BYTES: Final = 64 * 1024
@dataclass(frozen=True, slots=True)
class _SsePreface:
"""Complete leading SSE frames with no ``data:`` line, relayed verbatim before the stream shape is decided."""
"""Complete leading SSE frames that carry no data, relayed verbatim before the stream shape is decided."""
raw: bytes
@ -125,17 +126,27 @@ class _SsePreface:
_SSE_FRAME_END: Final = re.compile(rb"\r\n\r\n|\n\n|\r\r")
def _carries_data(frame: bytes) -> bool:
return any(line.startswith(b"data:") and line[len(b"data:") :].strip() for line in frame.splitlines())
def _split_sse_preface(complete_frames: bytes) -> tuple[bytes, bytes]:
"""Split complete frames into ``(frames before the first data-bearing frame, that frame and everything after)``."""
start = 0
for end in _SSE_FRAME_END.finditer(complete_frames):
frame = complete_frames[start : end.end()]
if any(line.startswith(b"data:") for line in frame.splitlines()):
if _carries_data(frame):
return complete_frames[:start], complete_frames[start:]
start = end.end()
return complete_frames, b""
def _holds_json_object_frame(frames: bytes) -> bool:
return any(
line.startswith(b"data:") and line[len(b"data:") :].lstrip().startswith(b"{") for line in frames.splitlines()
)
def _flush_unmaskable_buffer(all_chunks: list[ModelResponseStream]) -> Iterator[ModelResponseStream]:
"""Buffered chunks flushed unmasked when a mixed stream shape makes reconstruction impossible."""
if not all_chunks:
@ -150,12 +161,13 @@ def _flush_unmaskable_buffer(all_chunks: list[ModelResponseStream]) -> Iterator[
async def _coalesce_first_sse_frame(stream: AsyncIterator[object]) -> AsyncGenerator[object, None]:
"""
Relay leading data-less SSE frames (comment keepalives, events without a
``data:`` line) as they complete, and join raw ``bytes`` chunks until they
hold one complete SSE event with a data line, so the stream shape is
decided on a whole frame rather than a transport fragment. Everything
after that first frame is forwarded untouched. The byte cap can only be
reached by a single unterminated frame.
Relay leading data-less SSE frames (comment keepalives, events with no
data) as they complete, and join raw ``bytes`` chunks until they
hold one complete SSE event with a JSON object on its data line, so the
stream shape is decided on a whole frame rather than a transport fragment
or a line that says nothing about the shape. Everything after that frame
is forwarded untouched. The byte cap bounds a single unterminated frame
and a run of frames with no JSON object.
"""
pending = b""
try:
@ -169,7 +181,7 @@ async def _coalesce_first_sse_frame(stream: AsyncIterator[object]) -> AsyncGener
if preface:
yield _SsePreface(preface)
pending = classifiable + tail
if classifiable or len(pending) >= _MAX_FIRST_SSE_FRAME_BYTES:
if _holds_json_object_frame(classifiable) or len(pending) >= _MAX_FIRST_SSE_FRAME_BYTES:
break
else:
if pending:
@ -1523,7 +1535,7 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
first_frame_is_anthropic = (
not passthrough_due_to_unknown_stream_shape
and not all_chunks
and is_anthropic_sse_stream((chunk,))
and self._is_anthropic_first_frame(chunk)
)
if not first_frame_is_anthropic:
passthrough_due_to_unknown_stream_shape = (
@ -1565,13 +1577,29 @@ class _OPTIONAL_PresidioPIIMasking(CustomGuardrail):
for chunk in all_chunks:
yield chunk
def _is_anthropic_first_frame(self, chunk: bytes) -> bool:
if is_anthropic_sse_stream((chunk,)):
return True
if has_unreadable_json_event((chunk,)):
raise self._unreadable_stream_refusal()
return False
def _unreadable_stream_refusal(self) -> GuardrailRaisedException:
return GuardrailRaisedException(
guardrail_name=self.guardrail_name,
message="Presidio could not read the upstream stream; cannot verify PII when PII protection is configured",
should_wrap_with_default_message=False,
)
async def _mask_anthropic_sse_stream(
self, first_chunk: bytes, rest: AsyncIterator[object], request_data: dict
) -> tuple[object, ...]:
rest_chunks: Final = [chunk async for chunk in rest]
chunks: Final = (first_chunk, *rest_chunks)
assembled: Final = assemble_anthropic_sse_stream(chunks, restore_identity=True)
assembled: Final = assemble_anthropic_sse_stream(chunks, restore_identity=True, through_upstream_error=True)
if assembled is None:
if has_unreadable_json_event(chunks):
raise self._unreadable_stream_refusal()
verbose_proxy_logger.warning(
"Presidio apply_to_output: raw SSE stream could not be assembled into a response. "
"Output PII masking was skipped for this response."

View file

@ -25,8 +25,10 @@ from litellm.proxy.common_utils.callback_utils import (
)
from litellm.proxy.guardrails.anthropic_sse import (
anthropic_sse_chunks_from_response,
anthropic_sse_error_events,
assemble_anthropic_sse_stream,
is_raw_sse_stream,
is_sse_error_stream,
)
from litellm.types.guardrails import GuardrailEventHooks, LitellmParams
from litellm.types.proxy.guardrails.guardrail_hooks.tool_permission import (
@ -333,7 +335,7 @@ class ToolPermissionGuardrail(CustomGuardrail):
parsed_arguments = arguments
else:
return None, "arguments must be a JSON object"
except (json.JSONDecodeError, TypeError) as exc:
except (json.JSONDecodeError, TypeError, RecursionError) as exc:
verbose_proxy_logger.warning(
"Tool Permission Guardrail: Failed to decode arguments for tool %s: %s",
tool_call.function.name,
@ -957,9 +959,11 @@ class ToolPermissionGuardrail(CustomGuardrail):
yield chunk
return
anthropic_response: Final = assemble_anthropic_sse_stream(all_chunks)
anthropic_response: Final = assemble_anthropic_sse_stream(
all_chunks, restore_identity=bool(anthropic_sse_error_events(all_chunks)), through_upstream_error=True
)
if anthropic_response is None:
if is_raw_sse_stream(all_chunks):
if is_raw_sse_stream(all_chunks) and not is_sse_error_stream(all_chunks):
raise GuardrailRaisedException(
guardrail_name=self.guardrail_name,
message=(
@ -971,16 +975,24 @@ class ToolPermissionGuardrail(CustomGuardrail):
yield chunk
return
anthropic_denials: Final = self._check_assembled_stream(anthropic_response)
if not anthropic_denials:
for chunk in all_chunks:
yield chunk
return
self._modify_response_with_permission_errors(anthropic_response, anthropic_denials)
for sse_chunk in anthropic_sse_chunks_from_response(anthropic_response, all_chunks, keep_explanations=True):
for sse_chunk in self._anthropic_stream_to_deliver(anthropic_response, all_chunks):
yield sse_chunk
def _anthropic_stream_to_deliver(
self, anthropic_response: ModelResponse, source_chunks: Sequence[object]
) -> Sequence[object]:
"""The frames the client gets: on a failed upstream only what the check read, never the upstream's own.
Assembly reads past an upstream error, and a client can read a frame of a broken stream that assembly
does not, so forwarding those frames would pass a tool call nobody checked.
"""
denials: Final = self._check_assembled_stream(anthropic_response)
if denials:
self._modify_response_with_permission_errors(anthropic_response, denials)
elif not anthropic_sse_error_events(source_chunks):
return source_chunks
return anthropic_sse_chunks_from_response(anthropic_response, source_chunks, keep_explanations=True)
def _check_assembled_stream(
self, assembled: ModelResponse
) -> tuple[tuple[ChatCompletionMessageToolCall, PermissionError], ...]:

View file

@ -21,7 +21,7 @@ from litellm.proxy.auth.auth_utils import (
)
from litellm.proxy.auth.budget_throttle import throttled_limit
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit, socket_session_usage
from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup
from litellm.types.utils import CallTypes, Usage
@ -41,10 +41,12 @@ def _precise_minute(now: datetime) -> str:
return now.strftime("%Y-%m-%d-%H-%M")
def _response_total_tokens(response_obj: object) -> int:
if not isinstance(response_obj, (ModelResponse, EmbeddingResponse, TextCompletionResponse)):
return 0
response_usage: Final = getattr(response_obj, "usage", None)
def _response_total_tokens(kwargs: Mapping[str, object], response_obj: object) -> int:
response_usage: Final = (
getattr(response_obj, "usage", None)
if isinstance(response_obj, (ModelResponse, EmbeddingResponse, TextCompletionResponse))
else socket_session_usage(kwargs, response_obj)
)
return response_usage.total_tokens if isinstance(response_usage, Usage) else 0
@ -556,7 +558,10 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
precise_minute: Final = _precise_minute(self._clock())
total_tokens: int = _response_total_tokens(response_obj)
total_tokens: Final = _response_total_tokens(
kwargs, # pyright: ignore[reportUnknownArgumentType] # legacy callback kwargs are untyped
response_obj,
)
# ------------
# Update usage - API Key
@ -630,8 +635,6 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
# Update usage - User
# ------------
if user_api_key_user_id is not None:
total_tokens = _response_total_tokens(response_obj)
request_count_api_key = f"{user_api_key_user_id}::{precise_minute}::request_count"
current = await self.internal_usage_cache.async_get_cache(
@ -639,7 +642,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
litellm_parent_otel_span=litellm_parent_otel_span,
) or {
"current_requests": 1,
"current_tpm": total_tokens,
"current_tpm": 0,
"current_rpm": 1,
}
@ -656,8 +659,6 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
# Update usage - Team
# ------------
if user_api_key_team_id is not None:
total_tokens = _response_total_tokens(response_obj)
request_count_api_key = f"{user_api_key_team_id}::{precise_minute}::request_count"
current = await self.internal_usage_cache.async_get_cache(
@ -665,7 +666,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
litellm_parent_otel_span=litellm_parent_otel_span,
) or {
"current_requests": 1,
"current_tpm": total_tokens,
"current_tpm": 0,
"current_rpm": 1,
}
@ -682,8 +683,6 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
# Update usage - End User
# ------------
if user_api_key_end_user_id is not None:
total_tokens = _response_total_tokens(response_obj)
request_count_api_key = f"{user_api_key_end_user_id}::{precise_minute}::request_count"
current = await self.internal_usage_cache.async_get_cache(
@ -691,7 +690,7 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
litellm_parent_otel_span=litellm_parent_otel_span,
) or {
"current_requests": 1,
"current_tpm": total_tokens,
"current_tpm": 0,
"current_rpm": 1,
}

View file

@ -46,8 +46,10 @@ from litellm.constants import (
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
NO_MAX_TOKENS_TPM_RESERVATION_FRACTION,
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY,
)
from litellm.cost_calculator import get_usage_object
from litellm.exceptions import ServiceUnavailableError
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.add_retry_fallback_headers import (
ensure_response_additional_headers,
@ -85,7 +87,7 @@ from litellm.proxy.hooks.batch_enqueued_tokens import (
batch_response_view,
canonical_provider_batch_id,
)
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit, socket_session_usage
from litellm.proxy.spend_tracking.input_tokens import (
inline_pdf_file_count,
models_counted_for,
@ -745,6 +747,7 @@ class RequestRateLimiterStash:
batch_tpd_refund_ops: tuple[ReservationAwareIncrementOperation, ...] = ()
reservation_released: bool = False
tpm_limited_tags: frozenset[str] = field(default_factory=frozenset)
socket_agent_windows: tuple[RateLimitDescriptor, ...] = ()
@dataclass(frozen=True, slots=True)
@ -864,6 +867,62 @@ def _call_id_from_callback_kwargs(kwargs: object) -> str | None:
return call_id if isinstance(call_id, str) else None
def _socket_connection_call_id(kwargs: Mapping[str, object]) -> object:
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params"))
litellm_metadata: Final = None if litellm_params is None else as_str_mapping(litellm_params.get("litellm_metadata"))
return None if litellm_metadata is None else litellm_metadata.get(RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY)
def _is_socket_turn(kwargs: Mapping[str, object]) -> bool:
if kwargs.get("call_type") == CallTypes.aresponses_websocket.value:
return True
connection_stash: Final = get_request_stash()
connection_call_id: Final = _socket_connection_call_id(kwargs)
return (
connection_stash is not None
and connection_call_id is not None
and connection_call_id == connection_stash.owner_litellm_call_id
)
_AGENT_DESCRIPTOR_KEYS: Final = frozenset({"agent", "agent_session"})
_SOCKET_TURN_UNCOUNTED_DESCRIPTOR_KEYS: Final = frozenset(
{PROJECT_ITPM_DESCRIPTOR_KEY, PROJECT_OTPM_DESCRIPTOR_KEY, PTU_TEAM_DESCRIPTOR_KEY}
)
def _socket_turn_tpm_windows(descriptors: Sequence[RateLimitDescriptor]) -> tuple[RateLimitDescriptor, ...]:
return tuple(
RateLimitDescriptor(
key=descriptor["key"],
value=descriptor["value"],
rate_limit=RateLimitDescriptorRateLimitObject(
tokens_per_unit=tokens_per_unit, window_size=(descriptor.get("rate_limit") or {}).get("window_size")
),
)
for descriptor in descriptors
if descriptor["key"] not in _SOCKET_TURN_UNCOUNTED_DESCRIPTOR_KEYS
and (tokens_per_unit := (descriptor.get("rate_limit") or {}).get("tokens_per_unit")) is not None
)
def _unmodeled_response_usage(kwargs: Mapping[str, object], response_obj: object) -> Usage | None:
combined_usage: Final = kwargs.get("combined_usage_object")
if isinstance(combined_usage, Usage):
return combined_usage
return socket_session_usage(kwargs, response_obj)
def _tpm_limited_tags(call_stash: RequestRateLimiterStash | None, kwargs: Mapping[str, object]) -> frozenset[str]:
if call_stash is not None:
return call_stash.tpm_limited_tags
connection_stash: Final = get_request_stash()
if connection_stash is None or not _is_socket_turn(kwargs):
return frozenset()
return connection_stash.tpm_limited_tags
def _parse_output_cap_value(raw_value: object) -> int | None:
if isinstance(raw_value, bool) or not isinstance(raw_value, (int, float, str)):
return None
@ -2967,6 +3026,40 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
if response["overall_code"] == "OVER_LIMIT":
self._handle_rate_limit_error(response, descriptors, requested_model)
@with_service_target("rate_limits")
async def enforce_token_limits_for_frame(
self, user_api_key_dict: UserAPIKeyAuth, requested_model: str | None
) -> None:
connection_stash: Final = get_request_stash()
tags: Final = sorted(connection_stash.tpm_limited_tags) if connection_stash is not None else []
agent_windows: Final = connection_stash.socket_agent_windows if connection_stash is not None else ()
hidden_stash: Final = _request_stash.set(None)
try:
descriptors: Final = await self._build_request_rate_limit_descriptors(
user_api_key_dict,
MappingProxyType({"model": requested_model, "metadata": {"tags": tags}}),
CallTypes.aresponses_websocket.value,
)
finally:
_request_stash.reset(hidden_stash)
built_scopes: Final = frozenset((d["key"], d["value"]) for d in descriptors)
tpm_windows: Final = _socket_turn_tpm_windows(
(*descriptors, *(d for d in agent_windows if (d["key"], d["value"]) not in built_scopes))
)
try:
response: Final = await self.should_rate_limit(
tpm_windows,
parent_otel_span=user_api_key_dict.parent_otel_span,
read_only=True,
skip_parallel_check=True,
)
except RateLimitUnverifiableError as unverifiable:
raise ServiceUnavailableError(
message=RATE_LIMIT_UNVERIFIABLE_MESSAGE, llm_provider="litellm_proxy", model=requested_model or ""
) from unverifiable
if response["overall_code"] == "OVER_LIMIT":
self._handle_rate_limit_error(response, tpm_windows, requested_model)
def _rate_limited_model(self, requested_model: str | None) -> RateLimitedModel | None:
if not requested_model:
return None
@ -3777,7 +3870,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
def _handle_rate_limit_error(
self,
response: RateLimitResponse,
descriptors: list[RateLimitDescriptor],
descriptors: Sequence[RateLimitDescriptor],
requested_model: str | None = None,
) -> None:
"""Handle rate limit exceeded by raising :class:`ProxyRateLimitError` (a 429)."""
@ -4329,6 +4422,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
for d in descriptors
if d["key"] == "tag" and d["rate_limit"] is not None and d["rate_limit"].get("tokens_per_unit") is not None
)
stash.socket_agent_windows = tuple(d for d in descriptors if d["key"] in _AGENT_DESCRIPTOR_KEYS)
# Only check rate limits if we have descriptors with actual limits
if descriptors:
@ -5286,9 +5380,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
# Get total tokens from response. Responses LiteLLM does not model
# (e.g. pass-through, whose usage is reported by the upstream rather
# than parsed out of the body) carry their usage in
# ``combined_usage_object`` instead, and would otherwise never charge
# the TPM window.
_combined_usage: Final = kwargs.get("combined_usage_object")
# ``combined_usage_object`` instead (a Responses WebSocket turn in its
# raw events) and would otherwise never charge the TPM window.
_usage: Final[Usage | dict[str, object] | None] = (
getattr(response_obj, "usage", None)
if isinstance(
@ -5302,9 +5395,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
else get_usage_object(response_obj)
if isinstance(response_obj, ExtractionResponse)
else _combined_usage
if isinstance(_combined_usage, Usage)
else None
else _unmodeled_response_usage(kwargs, response_obj)
)
total_tokens = self._get_total_tokens_from_usage(usage=_usage, rate_limit_type=rate_limit_type)
if total_tokens == 0:
@ -5337,7 +5428,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
targets: Final = self._collect_tpm_scope_targets(
standard_logging_metadata=standard_logging_metadata,
kwargs=kwargs,
tpm_limited_tags=stash.tpm_limited_tags if stash is not None else frozenset(),
tpm_limited_tags=_tpm_limited_tags(stash, kwargs),
model_group=reconcile_model.group if reconcile_model is not None else None,
)
targets.extend(
@ -5365,16 +5456,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
reserved_tokens=reserved_tokens,
)
)
ptu_skips_socket_usage: Final = kwargs.get("call_type") == CallTypes.aresponses_websocket.value
pipeline_operations.extend(
self._build_team_ptu_tpm_ops(
standard_logging_metadata, # pyright: ignore[reportUnknownArgumentType] # untyped logging metadata
usage_source=response_obj if _usage is None else _usage,
usage_source=response_obj if _usage is None or ptu_skips_socket_usage else _usage,
reconcile_model=reconcile_model,
reserved_scopes=reserved_scopes,
reserved_ceiling=stash.ptu_ceiling if stash is not None else None,
reserved_tokens=stash.ptu_reserved_tokens if stash is not None else 0,
reserved_windows=stash.ptu_reserved_window_identities if stash is not None else frozenset(),
total_tokens=total_tokens,
total_tokens=0 if ptu_skips_socket_usage else total_tokens,
)
)
@ -5469,27 +5561,14 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
else None,
)
pipeline_operations: Final = self._build_success_event_pipeline_operations(
kwargs=kwargs,
response_obj=response_obj,
rate_limit_type=rate_limit_type,
await self._apply_usage_operations(
self._build_success_event_pipeline_operations(
kwargs=kwargs,
response_obj=response_obj,
rate_limit_type=rate_limit_type,
),
litellm_parent_otel_span,
)
unguarded_operations: Final = [
operation for operation in pipeline_operations if _as_window_guarded(operation) is None
]
window_guarded_operations: Final = tuple(
guarded for operation in pipeline_operations if (guarded := _as_window_guarded(operation)) is not None
)
if unguarded_operations:
await self.async_increment_tokens_with_ttl_preservation(
pipeline_operations=unguarded_operations,
parent_otel_span=litellm_parent_otel_span,
)
if window_guarded_operations:
await self.async_increment_reservation_aware_tokens(
pipeline_operations=window_guarded_operations,
parent_otel_span=litellm_parent_otel_span,
)
io_token_operations: Final = self._build_io_token_reservation_ops(
kwargs=kwargs,
response_obj=response_obj,
@ -5509,6 +5588,26 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
except Exception as e:
verbose_proxy_logger.exception("Error in rate limit success event: %s", e)
async def _apply_usage_operations(
self, pipeline_operations: Sequence[RedisPipelineIncrementOperation], parent_otel_span: Span | None
) -> None:
unguarded_operations: Final = [
operation for operation in pipeline_operations if _as_window_guarded(operation) is None
]
window_guarded_operations: Final = tuple(
guarded for operation in pipeline_operations if (guarded := _as_window_guarded(operation)) is not None
)
if unguarded_operations:
await self.async_increment_tokens_with_ttl_preservation(
pipeline_operations=unguarded_operations,
parent_otel_span=parent_otel_span,
)
if window_guarded_operations:
await self.async_increment_reservation_aware_tokens(
pipeline_operations=window_guarded_operations,
parent_otel_span=parent_otel_span,
)
@with_service_target("rate_limits")
async def async_logging_hook(
self,
@ -5622,11 +5721,12 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
try:
litellm_parent_otel_span: Final[Span | None] = get_parent_otel_span_from_kwargs(kwargs)
failed_call: Final = _callback_kwargs_mapping(kwargs)
litellm_parent_otel_span: Final[Span | None] = get_parent_otel_span_from_kwargs(failed_call)
pipeline_operations: Final[list[RedisPipelineIncrementOperation]] = []
stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs))
stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(failed_call))
await self._release_stashed_parallel_slot(stash, litellm_parent_otel_span, in_logging_callback=True)
# Skip the reservation refund if async_post_call_failure_hook
@ -5638,7 +5738,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
if stash is None or stash.reservation_released
else (stash.reserved_tokens, stash.itpm_reserved_tokens, stash.otpm_reserved_tokens)
)
tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(kwargs)
tpm_actual, itpm_actual, otpm_actual = self._recovered_partial_usage_tokens(failed_call)
if stash is not None and reserved_tokens > 0:
verbose_proxy_logger.debug(
@ -5722,6 +5822,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
if stash is not None and (reserved_tokens > 0 or itpm_reserved > 0 or otpm_reserved > 0):
stash.reservation_released = True
if stash is None and tpm_actual > 0 and _is_socket_turn(failed_call):
await self._apply_usage_operations(
self._build_success_event_pipeline_operations(
kwargs=dict(failed_call), response_obj=None, rate_limit_type=self.get_rate_limit_type()
),
litellm_parent_otel_span,
)
except Exception as e:
verbose_proxy_logger.exception("Error in rate limit failure event: %s", e)

View file

@ -2,14 +2,32 @@
Shared utility functions for rate limiter hooks.
"""
from collections.abc import Mapping
from typing import Final
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import PROXY_LLM_PROVIDER_FALLBACK
from litellm.cost_calculator import ResponsesWebSocketTokenUsageProcessor
from litellm.litellm_core_utils.core_helpers import iter_items
from litellm.types.router import ModelGroupInfo
from litellm.types.utils import PriorityReservationDict
from litellm.types.utils import CallTypes, PriorityReservationDict, Usage
_SOCKET_SESSION_EVENTS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
def socket_session_usage(kwargs: Mapping[str, object], response_obj: object) -> Usage | None:
if not isinstance(response_obj, list):
return None
match kwargs.get("call_type"):
case CallTypes.aresponses_websocket.value:
return ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results(
_SOCKET_SESSION_EVENTS.validate_python(response_obj)
)
case _:
return None
def resolve_llm_provider_for_rate_limit(

View file

@ -1,14 +1,15 @@
# What is this?
## Helper utils for the management endpoints (keys/users/teams)
from collections.abc import Callable, Mapping, MutableMapping, Sequence
from collections.abc import Awaitable, Callable, Mapping, MutableMapping, Sequence
from collections.abc import Set as AbstractSet
from datetime import datetime
from functools import wraps
from types import MappingProxyType
from typing import Any, Final, Protocol
from typing import Any, Final, ParamSpec, Protocol, TypeVar
from fastapi import HTTPException, Request
from pydantic import BaseModel, TypeAdapter
from starlette.exceptions import HTTPException as StarletteHTTPException
import litellm
from litellm._internal_context import with_service_target
@ -687,6 +688,7 @@ async def _emit_management_endpoint_otel_span(
end_time: datetime,
result: Any = None,
exception: Exception | None = None,
log_payload: bool = True,
) -> None:
"""Stamp + end the parent OTEL SERVER span for a management endpoint.
@ -716,8 +718,8 @@ async def _emit_management_endpoint_otel_span(
)
route = get_request_route(http_request)
request_body: dict = await read_request_body( # rebind-ok: pre-existing rebinding on a rename-only line
request=http_request
request_body: dict = ( # rebind-ok: pre-existing rebinding on a rename-only line
await read_request_body(request=http_request) if log_payload else {}
)
else:
route = func.__name__
@ -738,7 +740,7 @@ async def _emit_management_endpoint_otel_span(
)
_response: dict[str, object] | None = None
if exception is None and result is not None:
if log_payload and exception is None and result is not None:
try:
raw: Final[Mapping[str, object]] = dict(result)
_response = {k: v for k, v in raw.items() if k not in _CREDENTIAL_FIELDS}
@ -767,37 +769,60 @@ async def _emit_management_endpoint_otel_span(
)
def management_endpoint_wrapper(func):
_P: Final = ParamSpec("_P")
_R: Final = TypeVar("_R")
def management_endpoint_wrapper(func: Callable[_P, Awaitable[_R]]) -> Callable[_P, Awaitable[_R]]:
"""
This wrapper does the following:
1. Log I/O, Exceptions to OTEL
2. Create an Audit log for success calls
"""
return _wrap_management_endpoint(func, log_payload=True)
@wraps(func)
async def wrapper(*args, **kwargs):
def management_endpoint_span_wrapper(func: Callable[_P, Awaitable[_R]]) -> Callable[_P, Awaitable[_R]]:
return _wrap_management_endpoint(func, log_payload=False)
def _caller_auth(endpoint_kwargs: Mapping[str, object]) -> UserAPIKeyAuth:
caller: Final = endpoint_kwargs.get("user_api_key_dict")
return caller if isinstance(caller, UserAPIKeyAuth) else UserAPIKeyAuth()
def _proxy_exception_handler_ends_server_span(exception: Exception) -> bool:
return not isinstance(exception, StarletteHTTPException)
def _wrap_management_endpoint(
endpoint: Callable[_P, Awaitable[_R]], *, log_payload: bool
) -> Callable[_P, Awaitable[_R]]:
@wraps(endpoint)
async def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R:
start_time: Final = datetime.now()
try:
result: Final = await func(*args, **kwargs)
result: Final = await endpoint(*args, **kwargs)
end_time = datetime.now()
try:
user_api_key_dict: UserAPIKeyAuth = kwargs.get("user_api_key_dict") or UserAPIKeyAuth()
user_api_key_dict: UserAPIKeyAuth = _caller_auth(kwargs)
await send_management_endpoint_alert(
request_kwargs=kwargs,
user_api_key_dict=user_api_key_dict,
function_name=func.__name__,
function_name=endpoint.__name__,
)
parent_otel_span: Span | None = getattr(user_api_key_dict, "parent_otel_span", None)
if parent_otel_span is not None:
await _emit_management_endpoint_otel_span(
func=func,
func=endpoint,
kwargs=kwargs,
parent_otel_span=parent_otel_span,
start_time=start_time,
end_time=end_time,
result=result,
log_payload=log_payload,
)
# Delete updated/deleted info from cache
@ -813,17 +838,18 @@ def management_endpoint_wrapper(func):
except Exception as e:
end_time = datetime.now()
user_api_key_dict: UserAPIKeyAuth = kwargs.get("user_api_key_dict") or UserAPIKeyAuth()
user_api_key_dict: UserAPIKeyAuth = _caller_auth(kwargs)
parent_otel_span = getattr(user_api_key_dict, "parent_otel_span", None)
if parent_otel_span is not None:
if parent_otel_span is not None and (log_payload or not _proxy_exception_handler_ends_server_span(e)):
try:
await _emit_management_endpoint_otel_span(
func=func,
func=endpoint,
kwargs=kwargs,
parent_otel_span=parent_otel_span,
start_time=start_time,
end_time=end_time,
exception=e,
log_payload=log_payload,
)
except Exception as otel_exc:
# Non-Blocking Exception - never let OTEL failures swallow

View file

@ -438,6 +438,14 @@ def is_mcp_admitted_user_subject(user_api_key_auth: UserAPIKeyAuth | None) -> bo
_is_mcp_admitted_user_subject: Final = is_mcp_admitted_user_subject
def has_team_unbound_mcp_admin_view(user_api_key_auth: UserAPIKeyAuth | None) -> bool:
return (
user_api_key_auth is not None
and user_api_key_has_admin_view(user_api_key_auth)
and (not user_api_key_auth.team_id or user_api_key_auth.team_id == UI_TEAM_ID)
)
def _gateway_dcr_challenge_target(
route: str,
mcp_servers: list[str] | None,
@ -4290,19 +4298,19 @@ class MCPRequestHandler:
"""Whether this principal's admin-view role grants the unscoped MCP resolution, whatever
credential carries it (admin key, dashboard session, or OAuth-admitted session subject).
Two bounds disqualify, one per ownership of the row. A CREDENTIAL's explicit
``object_permission.mcp_servers`` scope wins even for admins, including the empty list. An
admitted subject's object_permission is the user's own row, whose ``mcp_servers`` column is
[] by DB default, so for that shape the row binds through the entitlement ceiling instead
(any non-empty entitlement, or an unresolved one, disqualifies), exactly as
``operator_open_server_ids`` reads the same row. The one owner of this predicate: the
server-axis registry resolution in ``get_allowed_mcp_servers`` and the tools-axis open
channel in ``resolve_admitted_subject_tools`` both consult it, so the two axes cannot
disagree."""
A credential on a real team never qualifies, a virtual key or a ``lite login`` session alike: its team
bounds its MCP access whoever owns it, as it bounds the credential's model access. Two more bounds disqualify, one per ownership of the row. A CREDENTIAL's
explicit ``object_permission.mcp_servers`` scope wins even for admins, including the empty list. An
admitted subject's object_permission is the user's own row, whose ``mcp_servers`` column is [] by DB
default, so for that shape the row binds through the entitlement ceiling instead (any non-empty
entitlement, or an unresolved one, disqualifies), exactly as ``operator_open_server_ids`` reads the same
row. The one owner of this predicate: the server-axis registry resolution in ``get_allowed_mcp_servers``
and the tools-axis open channel in ``resolve_admitted_subject_tools`` both consult it, so the two axes
cannot disagree."""
if (
user_api_key_auth is None
or user_api_key_auth.mcp_explicit_grants_only
or not user_api_key_has_admin_view(user_api_key_auth)
or not has_team_unbound_mcp_admin_view(user_api_key_auth)
):
return False
object_permission: Final = user_api_key_auth.object_permission

View file

@ -3705,7 +3705,7 @@ class MCPServerManager:
Priority:
1. If object_permission.mcp_servers is explicitly set, use it (even for admins)
2. If admin and no object_permission, return all servers
2. If admin, not bound to a real team, and no object_permission, return all servers
3. Otherwise, use standard permission checks
"""
if managed_agent_policy(user_api_key_auth) is not None:

View file

@ -223,6 +223,7 @@ __all__ = (
"mcp_get_prompt",
"mcp_read_resource",
"raise_denied_scoped_mcp_access",
"raise_generic_mcp_access_denial",
)
@ -763,8 +764,14 @@ async def raise_denied_scoped_mcp_access(
)
}
raise HTTPException(status_code=403, detail=group_denial)
raise_generic_mcp_access_denial(requested_names)
def raise_generic_mcp_access_denial(requested_names: Sequence[str]) -> NoReturn:
generic_denial: Final[_McpDeniedDetail] = {
"error": f"The key is not allowed to access the requested MCP servers: {', '.join(requested_names)}"
"error": (
f"The key is not allowed to access the requested MCP servers: {', '.join(dict.fromkeys(requested_names))}"
)
}
raise HTTPException(status_code=403, detail=generic_denial)

View file

@ -520,6 +520,7 @@ if MCP_AVAILABLE:
"mcp_get_prompt",
"mcp_read_resource",
"raise_denied_scoped_mcp_access",
"raise_generic_mcp_access_denial",
"redact_mcp_resource_url",
)
from mcp.server import Server
@ -1197,6 +1198,7 @@ if MCP_AVAILABLE:
apply_display_name_overrides,
filter_tools_by_allowed_tools,
raise_denied_scoped_mcp_access,
raise_generic_mcp_access_denial,
)
@contextlib.asynccontextmanager
@ -1824,10 +1826,10 @@ if MCP_AVAILABLE:
row capping it.
Raises HTTPException(403) unless the principal holds toolset_id through one
of its grant sources. Admins always pass.
of its grant sources. Admins pass unless the key is bound to a real team.
"""
from litellm.proxy._types import LiteLLM_ObjectPermissionTable
from litellm.proxy.management_endpoints.common_utils import user_api_key_has_admin_view
from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import has_team_unbound_mcp_admin_view
# A key scoped to no MCP servers opts out of every MCP path. Enforce it
# here too, since toolset scoping replaces mcp_servers and would otherwise
@ -1841,7 +1843,7 @@ if MCP_AVAILABLE:
)
acting: Final = await acting_user(user_api_key_auth)
is_admin: Final = user_api_key_has_admin_view(acting)
is_admin: Final = has_team_unbound_mcp_admin_view(acting)
if not is_admin and toolset_id not in await granted(acting):
raise HTTPException(
status_code=403,

View file

@ -52,7 +52,11 @@ from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException
from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming, is_guardrail_intervention
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.fallback_generalizations import match_routing_generalization
from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers
from litellm.llms.anthropic.common_utils import (
AnthropicModelInfo,
anthropic_pass_through_api_base,
merge_anthropic_beta_headers,
)
from litellm.llms.azure.passthrough.transformation import (
foreign_azure_deployment,
is_azure_body_model_inference_endpoint,
@ -1047,7 +1051,7 @@ async def anthropic_proxy_route(
"""
[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)
"""
base_target_url = os.getenv("ANTHROPIC_API_BASE") or os.getenv("ANTHROPIC_BASE_URL") or "https://api.anthropic.com"
base_target_url = anthropic_pass_through_api_base()
encoded_endpoint = httpx.URL(endpoint).path
# Ensure endpoint starts with '/' for proper URL construction

View file

@ -71,7 +71,11 @@ from litellm.litellm_core_utils.litellm_logging import get_masked_values
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.litellm_core_utils.redact_messages import should_redact_message_logging
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.llms.anthropic.common_utils import is_anthropic_messages_url
from litellm.llms.anthropic.common_utils import (
anthropic_accepts_user_id,
anthropic_upstream_user_id_limit,
is_anthropic_messages_url,
)
from litellm.llms.base_llm.managed_resources.utils import (
resolve_passthrough_managed_id_provider,
)
@ -148,6 +152,8 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
EndpointType,
PassthroughStandardLoggingPayload,
WebSocketSessionMeter,
WebSocketSessionUsage,
)
from litellm.types.utils import TRUSTED_CALLBACK_VARS_FIELD, CallTypes, Usage
@ -501,7 +507,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
url: httpx.URL,
headers: dict,
requested_query_params: dict | None = None,
_parsed_body: dict | None = None,
_parsed_body: Mapping[str, object] | None = None,
forward_multipart: bool = False,
) -> httpx.Response:
"""
@ -715,8 +721,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
# unenforced. Set after the client merge so a request body cannot supply its own budget.
#
# Only for the built-in provider routes. `get_model_from_request` returns
# None for a user-defined pass-through, deliberately: its body is forwarded
# verbatim, so `model` there names an UPSTREAM model rather than a
# None for a user-defined pass-through, deliberately: the client's `model`
# goes upstream unchanged, so it names an UPSTREAM model rather than a
# LiteLLM-managed one. Enforcement is therefore skipped on those routes, and
# charging a counter anyway would track spend that nothing can refuse, and
# would attribute it to a budget the operator scoped to a LiteLLM model that
@ -1185,6 +1191,36 @@ def _unfiltered_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
return headers
def _json_object(value: object) -> Mapping[str, object]:
return cast("Mapping[str, object]", value) if isinstance(value, Mapping) else MappingProxyType({}) # cast-ok: json
def _anthropic_user_id_to_forward(
body: Mapping[str, object],
url: str,
has_pre_call_guardrails: Callable[[Mapping[str, object]], bool],
) -> str | None:
user_id: Final = _json_object(body.get("metadata")).get("user_id")
if not isinstance(user_id, str):
return None
max_length: Final = anthropic_upstream_user_id_limit(url)
if max_length is None:
return None
if not anthropic_accepts_user_id(user_id, max_length):
verbose_proxy_logger.warning("Not forwarding metadata.user_id because the upstream would refuse it")
return None
if has_pre_call_guardrails(body):
verbose_proxy_logger.debug("Not forwarding metadata.user_id because a pre-call guardrail cannot scan it")
return None
return user_id
def _with_anthropic_user_id(body: Mapping[str, object] | None, user_id: str | None) -> Mapping[str, object] | None:
if body is None or user_id is None:
return body
return {**body, "metadata": {"user_id": user_id}}
def _eu_jev_client_body(custom_llm_provider: str | None, body: Mapping[str, object]) -> Mapping[str, object] | None:
return MappingProxyType(dict(body)) if custom_llm_provider == "eu_jev" else None
@ -1421,6 +1457,10 @@ async def pass_through_request(
request_method=getattr(request, "method", None),
cost_per_request=cost_per_request,
)
hooked_body: Final = _json_object(_parsed_body)
client_user_id: Final = _anthropic_user_id_to_forward(
hooked_body, str(url), proxy_logging_obj.request_has_pre_call_guardrails
)
kwargs = HttpPassThroughEndpointHelpers.init_kwargs_for_pass_through_endpoint( # rebind-ok: pre-existing rebinding on a rename-only line
user_api_key_dict=user_api_key_dict,
_parsed_body=_parsed_body,
@ -1588,11 +1628,13 @@ async def pass_through_request(
else:
logging_url = str(url) + "?" + requested_query_params_str
outgoing_body: Final = cast("Mapping[str, object] | None", _parsed_body) # cast-ok: json
upstream_body: Final = _with_anthropic_user_id(outgoing_body, client_user_id)
logging_obj.pre_call(
input=[{"role": "user", "content": safe_dumps(_parsed_body)}],
api_key="",
additional_args={
"complete_input_dict": _parsed_body,
"complete_input_dict": upstream_body,
"api_base": str(logging_url),
"headers": redact_credential_headers(upstream_headers),
},
@ -1632,7 +1674,7 @@ async def pass_through_request(
url,
params=requested_query_params,
headers=upstream_headers,
json=_parsed_body,
json=upstream_body,
)
)
@ -1731,7 +1773,7 @@ async def pass_through_request(
url=url,
headers=upstream_headers,
requested_query_params=requested_query_params,
_parsed_body=_parsed_body,
_parsed_body=upstream_body,
forward_multipart=is_multipart,
)
verbose_proxy_logger.debug("response.headers= %s", response.headers)
@ -2595,6 +2637,13 @@ def _with_trace_context(headers: Mapping[str, str], parent_span: object) -> dict
return inject_trace_context(headers, parent_span=parent_span)
def _log_session_usage(logging_obj: LiteLLMLoggingObj, usage: WebSocketSessionUsage) -> None:
logging_obj.model = usage.model # rebind-ok: the spend logger reads model and cost off the shared logging object
logging_obj.model_call_details["model"] = usage.model # rebind-ok: same shared logging object
logging_obj.model_call_details["custom_llm_provider"] = usage.custom_llm_provider # rebind-ok: same
logging_obj.model_call_details["response_cost"] = usage.response_cost # rebind-ok: same
async def websocket_passthrough_request(
websocket: WebSocket,
target: str,
@ -2607,6 +2656,7 @@ async def websocket_passthrough_request(
setup_model_rewriter: Callable[[str], str] | None = None,
builtin_provider_route: bool = False,
model: str | None = None,
meter: WebSocketSessionMeter | None = None,
):
"""
WebSocket passthrough request handler.
@ -2622,6 +2672,7 @@ async def websocket_passthrough_request(
setup_model_rewriter: Optional rewrite of the setup frame's model before it reaches the upstream
builtin_provider_route: Whether a built-in provider route serves the connection, so its URL names the provider
model: The model the caller named before connecting, logged on a failure before the setup frame arrives
meter: Prices the session from the client text frames delivered upstream, for an upstream that reports no usage
"""
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.proxy.proxy_server import proxy_config, proxy_logging_obj
@ -2632,6 +2683,7 @@ async def websocket_passthrough_request(
# Initialize tracking variables
start_time: Final = datetime.now()
websocket_messages: Final[list[dict[str, object]]] = []
metered_total = 0 # rebind-ok: forward_client_to_upstream adds the measure of each client text frame it delivers
litellm_call_id: Final = str(uuid.uuid4())
verbose_proxy_logger.info("WebSocket passthrough (%s): Starting WebSocket connection to %s", endpoint, target)
@ -2750,6 +2802,7 @@ async def websocket_passthrough_request(
async def forward_client_to_upstream() -> None:
"""Forward messages from client to upstream WebSocket"""
nonlocal metered_total
try:
while True:
message = await websocket.receive()
@ -2819,6 +2872,8 @@ async def websocket_passthrough_request(
# Not a JSON message or doesn't contain setup data
await upstream_ws.send(_rewrite_vertex_live_setup_model(text_data, setup_model_rewriter))
if meter is not None:
metered_total += meter.measure(text_data)
elif bytes_data is not None:
await upstream_ws.send(bytes_data)
except asyncio.CancelledError:
@ -2945,6 +3000,8 @@ async def websocket_passthrough_request(
self.url = target_url
mock_response: Final = MockWebSocketResponse(target)
if meter is not None:
_log_session_usage(logging_obj, meter.price(metered_total))
# Use the same success handler as HTTP passthrough endpoints
GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(

View file

@ -181,6 +181,7 @@ from litellm.proxy.common_utils.token_count_settings import (
)
from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog
from litellm.proxy.management_helpers.model_group_custom_fields import model_group_custom_fields
from litellm.proxy.management_helpers.utils import management_endpoint_span_wrapper
from litellm.repositories import database as database_runtime
from litellm.router_strategy.auto_router.auto_router_model_naming import (
GATED_AUTO_ROUTER_CAPABILITIES,
@ -12339,6 +12340,7 @@ async def _deployment_hidden_by_listing_callbacks(deployment: Deployment, user_a
@router.get(
"/models", dependencies=[Depends(user_api_key_auth)], tags=["model management"]
) # if project requires model list
@management_endpoint_span_wrapper
async def model_list(
request: Request = None, # pyright: ignore[reportArgumentType] # FastAPI always injects the Request; the None default only serves direct in-process callers
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
@ -12649,6 +12651,7 @@ async def model_list(
dependencies=[Depends(user_api_key_auth)],
tags=["model management"],
)
@management_endpoint_span_wrapper
async def model_info(
model_id: str,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
@ -21985,9 +21988,9 @@ async def toolset_mcp_route(toolset_name: str, request: Request):
Namespace a toolset as its own MCP endpoint.
Connecting to /toolset/<name>/mcp exposes exactly the tools defined in
the toolset. Access is enforced: non-admin API keys must have the toolset
listed in their object_permission.mcp_toolsets grant list, or the request
will be rejected with a 403.
the toolset. Access is enforced: API keys other than an admin's key with no
team must hold the toolset through one of their grant sources, or the
request will be rejected with a 403.
"""
try:
from litellm.proxy.mcp_server.mcp_server_manager import (

View file

@ -3343,6 +3343,34 @@
],
"default_model_placeholder": "prism/deepseek-v4.1-flash"
},
{
"provider": "HOSTINGER_ROUTER",
"provider_display_name": "Hostinger Router",
"litellm_provider": "hostinger_router",
"credential_fields": [
{
"key": "api_base",
"label": "API Base",
"placeholder": "https://router.hostinger.com/v1",
"tooltip": null,
"required": false,
"field_type": "text",
"options": null,
"default_value": null
},
{
"key": "api_key",
"label": "API Key",
"placeholder": null,
"tooltip": null,
"required": true,
"field_type": "password",
"options": null,
"default_value": null
}
],
"default_model_placeholder": "hostinger_router/claude-sonnet-5"
},
{
"provider": "RECRAFT",
"provider_display_name": "Recraft",

View file

@ -36,7 +36,7 @@ from litellm._logging import verbose_proxy_logger
from litellm.anthropic_interface.exceptions.exception_mapping_utils import AnthropicExceptionMapping
from litellm.caching.dual_cache import DualCache
from litellm.constants import EMPTY_MAPPING, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, TERMINAL_RESPONSE_STATUSES
from litellm.exceptions import EndpointNotSupportedByModelError
from litellm.exceptions import EndpointNotSupportedByModelError, RateLimitError
from litellm.integrations.custom_guardrail import ModifyResponseException, guardrail_request_data_with_streaming
from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata_variable_name_from_kwargs
from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background
@ -103,6 +103,7 @@ from litellm.responses.streaming_iterator import (
ResponsesWsFrameGuard,
ResponsesWsFrameRejection,
spawn_logging_task,
turn_refusal_frame,
)
from litellm.router import Router
from litellm.types.llms.base import LiteLLMBaseModel
@ -2627,6 +2628,12 @@ async def _responses_websocket_session(
route_type="_aresponses_websocket",
skip_guardrails=True,
)
except RateLimitError as e:
verbose_proxy_logger.info("Responses WebSocket refused at open by a rate limit: %s", e)
with contextlib.suppress(WebSocketDisconnect, RuntimeError, OSError):
await websocket.send_text(turn_refusal_frame(e))
await websocket.close(code=1008, reason="Rate limit exceeded")
return
except Exception as e:
verbose_proxy_logger.exception("Responses WebSocket pre-call error")
try:

View file

@ -3031,14 +3031,21 @@ class ProxyLogging:
hook to consult, but it judges the payload the same way, so a proxy configured only with
one of those still has something to say about every record.
"""
if request_metadata.get("_guardrail_pipelines"):
return self.request_has_pre_call_guardrails({"metadata": dict(request_metadata)}, event_type)
def request_has_pre_call_guardrails(
self,
request_data: Mapping[str, object],
event_type: GuardrailEventHooks = GuardrailEventHooks.pre_call,
) -> bool:
if _policy_state_metadata(request_data).get("_guardrail_pipelines"):
return True
caps: Final = ProxyLogging.callback_capabilities()
if caps.has_content_enforcer:
return True
probe: Final = {"metadata": dict(request_metadata)}
return any(
isinstance(callback, CustomGuardrail) and callback.should_run_guardrail(data=probe, event_type=event_type)
isinstance(callback, CustomGuardrail)
and callback.should_run_guardrail(data=request_data, event_type=event_type)
for callback in caps.resolved_callbacks
)

View file

@ -10,6 +10,7 @@ from pydantic import TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
from litellm.responses.additional_tools import HoistedAdditionalTools, hoist_additional_tools
from litellm.responses.litellm_completion_transformation.client_tool_search import bridge_client_tool_search
from litellm.responses.litellm_completion_transformation.compaction_items import holds_compaction_summary
@ -148,6 +149,9 @@ class LiteLLMCompletionTransformationHandler:
responses_api_request=bridged_request,
custom_llm_provider=custom_llm_provider,
litellm_metadata=kwargs.get("litellm_metadata", {}),
tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(
_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)
),
)
raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}")
@ -240,5 +244,8 @@ class LiteLLMCompletionTransformationHandler:
responses_api_request=responses_api_request,
custom_llm_provider=litellm_completion_request.get("custom_llm_provider"),
litellm_metadata=kwargs.get("litellm_metadata", {}),
tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(
_REQUEST_KWARGS_ADAPTER.validate_python(kwargs)
),
)
raise ValueError(f"Unexpected response type: {type(litellm_completion_response)}")

View file

@ -200,6 +200,7 @@ class ResponsesSessionHandler:
previous_response_id: str,
item_cache_control: bool = False,
client_response_ids: tuple[str, ...] = (),
tool_call_provider_fields: bool = False,
stored_turns: StoredTurns | None = None,
) -> ChatCompletionSession:
"""
@ -241,9 +242,13 @@ class ResponsesSessionHandler:
for index in range(start.index, replayed_turns)
],
item_cache_control,
tool_call_provider_fields,
)
history_before_replay: Final = ResponsesSessionHandler._history_of_turns(
all_spend_logs[: start.index], proxy_server_requests[: start.index], item_cache_control
all_spend_logs[: start.index],
proxy_server_requests[: start.index],
item_cache_control,
tool_call_provider_fields,
)
chat_completion_message_history: Final = (
list(with_stored_calls_the_input_answers(replayed_history, history_before_replay))
@ -305,6 +310,7 @@ class ResponsesSessionHandler:
spend_logs: Sequence["SpendLogsPayload"],
proxy_server_requests: Sequence[Mapping[str, object] | None],
item_cache_control: bool,
tool_call_provider_fields: bool,
) -> list["SessionMessage"]:
return list(
chain.from_iterable(
@ -313,6 +319,7 @@ class ResponsesSessionHandler:
proxy_server_request_dict=proxy_server_request,
chat_completion_message_history=[],
item_cache_control=item_cache_control,
tool_call_provider_fields=tool_call_provider_fields,
)
for spend_log, proxy_server_request in zip(spend_logs, proxy_server_requests, strict=True)
)
@ -330,6 +337,7 @@ class ResponsesSessionHandler:
| Message
],
item_cache_control: bool = False,
tool_call_provider_fields: bool = False,
):
"""
Extend the chat completion message history with the spend log payload
@ -362,6 +370,7 @@ class ResponsesSessionHandler:
responses_api_request={},
replay_reasoning=True,
item_cache_control=item_cache_control,
tool_call_provider_fields=tool_call_provider_fields,
only_since_compaction=True,
)
)
@ -377,6 +386,7 @@ class ResponsesSessionHandler:
responses_api_request={},
replay_reasoning=True,
item_cache_control=item_cache_control,
tool_call_provider_fields=tool_call_provider_fields,
only_since_compaction=True,
)
)

View file

@ -9,6 +9,7 @@ import litellm
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_items
from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params, served_from_cache
from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend
from litellm.llms.anthropic.messages.native.utils import openai_chat_refusal_text
from litellm.main import stream_chunk_builder
from litellm.responses.litellm_completion_transformation.compaction_items import compaction_output_item
from litellm.responses.litellm_completion_transformation.custom_tools import (
@ -35,6 +36,7 @@ from litellm.types.llms.openai import (
ContentPartAddedEvent,
ContentPartDoneEvent,
ContentPartDonePartOutputText,
ContentPartDonePartRefusal,
FunctionCallArgumentsDeltaEvent,
FunctionCallArgumentsDoneEvent,
OutputItemAddedEvent,
@ -45,6 +47,8 @@ from litellm.types.llms.openai import (
ReasoningSummaryPartDoneEvent,
ReasoningSummaryTextDeltaEvent,
ReasoningSummaryTextDoneEvent,
RefusalDeltaEvent,
RefusalDoneEvent,
ResponseCompletedEvent,
ResponseCreatedEvent,
ResponseIncompleteEvent,
@ -88,6 +92,20 @@ def _output_items_with_id(items: tuple[Any, ...], item_type: str, item_id: str |
)
def _tool_call_objects(tool_calls: object) -> tuple[object, ...]:
return tuple(tool_calls) if isinstance(tool_calls, list) else ()
def _tool_call_provider_specific_fields(tool_call: object) -> Mapping[str, object]:
mapping: Final = as_str_mapping(tool_call)
fields: Final[object] = (
mapping.get("provider_specific_fields")
if mapping is not None
else getattr(tool_call, "provider_specific_fields", None)
)
return {"provider_specific_fields": fields} if fields else {}
def _is_signed_thinking_block(block: object) -> bool:
fields: Final = as_str_mapping(block)
return fields is not None and isinstance(block, dict) and bool(fields.get("signature") or fields.get("data"))
@ -110,6 +128,7 @@ def _delta_carries_output(delta: ChatCompletionDelta) -> bool:
delta.function_call,
getattr(delta, "images", None),
getattr(delta, "audio", None),
openai_chat_refusal_text(delta),
)
return any(fields) or _delta_has_signed_thinking_block(delta)
@ -119,6 +138,13 @@ def _delta_carries_reasoning_and_answer(delta: ChatCompletionDelta) -> bool:
return has_reasoning and bool(delta.content or delta.tool_calls)
_WEB_SEARCH_PROGRESS_EVENTS: Final = (
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, WebSearchCallInProgressEvent),
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, WebSearchCallSearchingEvent),
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),
)
class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
"""
Async iterator for processing streaming responses from the Responses API.
@ -132,6 +158,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
responses_api_request: ResponsesAPIOptionalRequestParams,
custom_llm_provider: str | None = None,
litellm_metadata: dict | None = None,
tool_call_provider_fields: bool = False,
):
self.model: str = model
self.litellm_custom_stream_wrapper: litellm.CustomStreamWrapper = litellm_custom_stream_wrapper
@ -139,6 +166,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
self.responses_api_request: ResponsesAPIOptionalRequestParams = responses_api_request
self.custom_llm_provider: str | None = custom_llm_provider
self.litellm_metadata: dict | None = litellm_metadata or {}
self.tool_call_provider_fields: Final = tool_call_provider_fields
# Store lightweight dict snapshots for stream_chunk_builder to reduce
# repeated Pydantic attribute access in end-of-stream assembly.
self.collected_chat_completion_chunks: list[dict[str, object]] = []
@ -152,6 +180,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
self.sent_output_text_done_event: bool = False
self.sent_output_content_part_done_event: bool = False
self.sent_output_item_done_event: bool = False
self._sent_refusal_events: bool = False
self._annotation_index: int = 0
self._annotations_awaiting_message: tuple[ChatCompletionAnnotation, ...] = ()
self.litellm_model_response: ModelResponse | TextCompletionResponse | None = None
@ -490,10 +519,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
except Exception:
tool_calls = None
if not tool_calls or not isinstance(tool_calls, list):
return
for tc in tool_calls:
for tc in _tool_call_objects(tool_calls):
call_id_raw = tc.get("id") if isinstance(tc, dict) else getattr(tc, "id", None)
if not call_id_raw:
continue
@ -568,7 +594,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
output_index=output_index,
sequence_number=self._sequence_number,
item=BaseLiteLLMOpenAIResponseObject(**item_kwargs),
item=BaseLiteLLMOpenAIResponseObject(
**item_kwargs,
**(_tool_call_provider_specific_fields(tc) if self.tool_call_provider_fields else {}),
),
)
self._pending_tool_events.append(item_done_event)
@ -592,11 +621,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
}
),
)
for event_type, event_class in (
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_IN_PROGRESS, WebSearchCallInProgressEvent),
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_SEARCHING, WebSearchCallSearchingEvent),
(ResponsesAPIStreamEvents.WEB_SEARCH_CALL_COMPLETED, WebSearchCallCompletedEvent),
):
progress_events: Final = (
_WEB_SEARCH_PROGRESS_EVENTS[:1] if item.status == "in_progress" else _WEB_SEARCH_PROGRESS_EVENTS
)
for event_type, event_class in progress_events:
self._sequence_number += 1
event = event_class(type=event_type, output_index=output_index, item_id=item.id)
event.__dict__["sequence_number"] = self._sequence_number
@ -798,9 +826,14 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
return event
def _queue_message_item_added_events(self) -> None:
if not self.sent_message_item_added_event:
self._queue_message_item_added_event()
self.sent_content_part_added_event = True
self._pending_response_events.append(self.create_content_part_added_event())
def _queue_message_item_added_event(self) -> None:
item_id: Final = self._message_item_id()
self.sent_message_item_added_event = True
self.sent_content_part_added_event = True
self._sequence_number += 1
event: Final = OutputItemAddedEvent(
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
@ -817,7 +850,47 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
)
event.__dict__["sequence_number"] = self._sequence_number
self._pending_response_events.append(event)
self._pending_response_events.append(self.create_content_part_added_event())
def _queue_refusal_events(self, refusal: str) -> None:
item_id: Final = self._message_item_id()
output_index: Final = self._message_index()
content_index: Final = 1 if self.sent_content_part_added_event else 0
events: Final = (
ContentPartAddedEvent(
type=ResponsesAPIStreamEvents.CONTENT_PART_ADDED,
item_id=item_id,
output_index=output_index,
content_index=content_index,
part=ContentPartDonePartRefusal(type="refusal", refusal=""),
),
RefusalDeltaEvent(
type=ResponsesAPIStreamEvents.REFUSAL_DELTA,
item_id=item_id,
output_index=output_index,
content_index=content_index,
delta=refusal,
),
RefusalDoneEvent(
type=ResponsesAPIStreamEvents.REFUSAL_DONE,
item_id=item_id,
output_index=output_index,
content_index=content_index,
refusal=refusal,
),
ContentPartDoneEvent(
type=ResponsesAPIStreamEvents.CONTENT_PART_DONE,
item_id=item_id,
output_index=output_index,
content_index=content_index,
part=ContentPartDonePartRefusal(type="refusal", refusal=refusal),
),
)
first_sequence_number: Final = self._sequence_number + 1
self._sequence_number += len(events)
self._pending_response_events.extend(
event.model_copy(update={"sequence_number": sequence_number})
for sequence_number, event in enumerate(events, start=first_sequence_number)
)
def _merge_provider_specific_fields(self, src: dict) -> None:
"""Merge provider_specific_fields using last-value-wins for lists.
@ -974,14 +1047,22 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
if self._cached_item_id is None:
self._cached_item_id = f"msg_{uuid.uuid4()}"
text: Final = litellm_complete_object.choices[0].message.content or ""
annotations = getattr(litellm_complete_object.choices[0].message, "annotations", None)
message: Final = litellm_complete_object.choices[0].message
text: Final = message.content or ""
annotations = getattr(message, "annotations", None)
response_annotations: Final = (
LiteLLMCompletionResponsesConfig.transform_chat_completion_annotations_to_response_output_annotations(
annotations=annotations
)
)
text_parts: Final = (
({"type": "output_text", "text": text, "annotations": response_annotations},)
if self.sent_content_part_added_event
else ()
)
refusal: Final = openai_chat_refusal_text(message)
refusal_parts: Final = ({"type": "refusal", "refusal": refusal},) if refusal is not None else ()
return OutputItemDoneEvent(
type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE,
output_index=self._message_index(),
@ -992,13 +1073,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
"status": "completed",
"type": "message",
"role": "assistant",
"content": [
{
"type": "output_text",
"text": text,
"annotations": response_annotations,
}
],
"content": [*text_parts, *refusal_parts],
}
),
)
@ -1050,22 +1125,33 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
def return_default_done_events(
self, litellm_complete_object: ModelResponse
) -> BaseLiteLLMOpenAIResponseObject | None:
message: Final = litellm_complete_object.choices[0].message
refusal: Final = openai_chat_refusal_text(message)
if self.sent_message_item_added_event is False:
final_content: Final = litellm_complete_object.choices[0].message.content or ""
if not final_content:
final_content: Final = message.content or ""
if not final_content and refusal is None:
self.sent_output_text_done_event = True
self.sent_output_content_part_done_event = True
self.sent_output_item_done_event = True
return None
self._queue_message_item_added_events()
if final_content:
self._queue_message_item_added_events()
else:
self._queue_message_item_added_event()
return self._pending_response_events.pop(0)
if self.sent_output_text_done_event is False:
if self.sent_output_text_done_event is False and self.sent_content_part_added_event:
self.sent_output_text_done_event = True
return self.create_output_text_done_event(litellm_complete_object)
if self.sent_output_content_part_done_event is False:
if self.sent_output_content_part_done_event is False and self.sent_content_part_added_event:
self.sent_output_content_part_done_event = True
return self.create_output_content_part_done_event(litellm_complete_object)
if refusal is not None and not self._sent_refusal_events:
self._sent_refusal_events = True
self._queue_refusal_events(refusal)
return self._pending_response_events.pop(0)
if self.sent_output_item_done_event is False:
self.sent_output_text_done_event = True
self.sent_output_content_part_done_event = True
self.sent_output_item_done_event = True
return self.create_output_item_done_event(litellm_complete_object)
return None
@ -1169,7 +1255,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
# DO NOT create message item
return
# Default: message
if openai_chat_refusal_text(delta) is not None and not delta.content:
self._queue_message_item_added_event()
return
self._queue_message_item_added_events()
return
@ -1413,7 +1501,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator):
# Priority 2: Handle text deltas
delta_content: Final = self._get_delta_string_from_streaming_choices(chunk.choices)
if delta_content:
if not self.sent_message_item_added_event:
if not self.sent_content_part_added_event:
self._queue_message_item_added_events()
self._hold_tool_calls_behind_text(getattr(chunk.choices[0].delta, "tool_calls", None))
self._sequence_number += 1

View file

@ -8,7 +8,7 @@ import uuid
from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from functools import partial
from itertools import accumulate, chain, groupby, islice
from itertools import accumulate, chain, groupby
from types import MappingProxyType
from typing import (
TYPE_CHECKING,
@ -28,7 +28,7 @@ from openai.types.chat.chat_completion_named_tool_choice_param import (
from openai.types.chat.chat_completion_named_tool_choice_param import (
Function as NamedToolChoiceFunction,
)
from openai.types.responses import ResponseFunctionToolCall, ResponseFunctionWebSearch
from openai.types.responses import ResponseFunctionToolCall, ResponseFunctionWebSearch, ResponseOutputRefusal
from openai.types.responses.response_compaction_item import ResponseCompactionItem
from openai.types.responses.response_create_params import ResponseInputParam
from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam
@ -45,8 +45,16 @@ from litellm.constants import (
REDACTED_BY_LITELLM,
REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER,
)
from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params
from litellm.integrations.anthropic_cache_control_hook import (
AnthropicCacheControlHook,
block_with_cache_control,
cache_control_count,
dropped_cache_control,
newest_marks_within_cache_control_limit,
whole_messages_within_cache_control_limit,
with_cache_control,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params
from litellm.litellm_core_utils.get_supported_openai_params import (
get_supported_openai_params,
)
@ -56,7 +64,9 @@ from litellm.litellm_core_utils.hidden_params import (
set_hidden_params,
)
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
from litellm.llms.anthropic.common_utils import supports_anthropic_cache_control
from litellm.llms.anthropic.messages.native.utils import anthropic_stop_reason, openai_chat_refusal_text
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.bedrock.common_utils import bedrock_declared_cache_point_support
from litellm.llms.openrouter.chat.transformation import openrouter_model_supports_cache_control
@ -365,8 +375,9 @@ class _ResponsesToolCallItem(Protocol):
def get(self, key: str, /) -> object: ...
class _MarkedToolCallChunk(ChatCompletionToolCallChunk):
class _ReplayedToolCallChunk(ChatCompletionToolCallChunk, total=False):
cache_control: ReadOnly[ChatCompletionCachedContent]
provider_specific_fields: ReadOnly[Mapping[str, object]]
class _MarkedToolMessage(ChatCompletionToolMessage):
@ -438,6 +449,32 @@ def _input_item_role(input_item: Mapping[str, object]) -> str:
return cast(str, input_item.get("role") or "user") # cast-ok: client-supplied role forwarded verbatim, unvalidated
def _starts_a_turn(input_item: Mapping[str, object]) -> bool:
item_type: Final = input_item.get("type")
if item_type in TOOL_CALL_OUTPUT_ITEM_TYPES:
return True
return item_type in (None, "message") and input_item.get("role") != "assistant"
def _is_unrun_hosted_search(input_item: Mapping[str, object]) -> bool:
return input_item.get("type") == "web_search_call" and input_item.get("status") == "in_progress"
def _without_encrypted_content(input_item: Mapping[str, object]) -> Mapping[str, object]:
return {key: value for key, value in input_item.items() if key != "encrypted_content"}
def _without_paused_turn_thinking(input_items: Sequence[Mapping[str, object]]) -> tuple[Mapping[str, object], ...]:
turns: Final = tuple(accumulate(int(_starts_a_turn(item)) for item in input_items))
paused_turns: Final = frozenset(
turn for turn, item in zip(turns, input_items, strict=True) if _is_unrun_hosted_search(item)
)
return tuple(
_without_encrypted_content(item) if turn in paused_turns and item.get("type") == "reasoning" else item
for turn, item in zip(turns, input_items, strict=True)
)
SessionMessage: TypeAlias = (
AllMessageValues
| GenericChatCompletionMessage
@ -603,21 +640,28 @@ def _validated_cache_control(value: object) -> ChatCompletionCachedContent | Non
return None
def _dropped_cache_control(_: object) -> None:
return None
def _without_cache_control_key(item: object) -> object:
def _without_keys(item: object, keys: frozenset[str]) -> object:
if not isinstance(item, dict):
return item
return {key: value for key, value in _STR_KEY_DICT_ADAPTER.validate_python(item).items() if key != "cache_control"}
return {key: value for key, value in _STR_KEY_DICT_ADAPTER.validate_python(item).items() if key not in keys}
def without_item_cache_control(input: str | ResponseInputParam) -> str | ResponseInputParam:
if not isinstance(input, list):
def _without_item_keys(input: str | ResponseInputParam, keys: frozenset[str]) -> str | ResponseInputParam:
if not keys or not isinstance(input, list):
return input
return cast( # cast-ok: the same input items, each minus its own cache_control key
ResponseInputParam, [_without_cache_control_key(item) for item in input]
return cast( # cast-ok: the same input items, each minus the given keys
ResponseInputParam, [_without_keys(item, keys) for item in input]
)
def _dropped_item_keys(item_cache_control: bool, tool_call_provider_fields: bool) -> frozenset[str]:
return frozenset(
key
for key, kept in (
("cache_control", item_cache_control),
("provider_specific_fields", tool_call_provider_fields),
)
if not kept
)
@ -694,83 +738,6 @@ def _content_blocks_carrying(content: object, cache_control: ChatCompletionCache
return _with_cache_control_on_last_block(_OBJECT_LIST_ADAPTER.validate_python(content), cache_control)
_MARK_CARRYING_MESSAGE_FIELDS: Final = ("content", "tool_calls")
def _mark_carrying_blocks(fields: Mapping[str, object]) -> Sequence[object]:
lists: Final = tuple(fields.get(key) for key in _MARK_CARRYING_MESSAGE_FIELDS)
return tuple(
chain.from_iterable(_OBJECT_LIST_ADAPTER.validate_python(value) for value in lists if isinstance(value, list))
)
def _block_carries_cache_control(block: object) -> bool:
return isinstance(block, dict) and _STR_KEY_DICT_ADAPTER.validate_python(block).get("cache_control") is not None
def _cache_control_count(message: object) -> int:
if not isinstance(message, dict):
return 0
fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message)
block_marks: Final = sum(1 for block in _mark_carrying_blocks(fields) if _block_carries_cache_control(block))
return int(fields.get("cache_control") is not None) + block_marks
_CacheControlRewrite: TypeAlias = Callable[[object], ChatCompletionCachedContent | None]
def _fields_with_cache_control(fields: Mapping[str, object], rewrite: _CacheControlRewrite) -> Mapping[str, object]:
others: Final = {key: value for key, value in fields.items() if key != "cache_control"}
if "cache_control" not in fields:
return others
cache_control: Final = rewrite(fields["cache_control"])
return others if cache_control is None else {**others, "cache_control": cache_control}
def _block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> object:
if not isinstance(block, dict):
return block
return _fields_with_cache_control(_STR_KEY_DICT_ADAPTER.validate_python(block), rewrite)
def _with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT:
if not isinstance(message, dict):
return message
fields: Final = _STR_KEY_DICT_ADAPTER.validate_python(message)
blocks_rewritten: Final = {
key: (
[_block_with_cache_control(block, rewrite) for block in _OBJECT_LIST_ADAPTER.validate_python(value)]
if key in _MARK_CARRYING_MESSAGE_FIELDS and isinstance(value, list)
else value
)
for key, value in fields.items()
}
return cast( # cast-ok: the same message shape with each cache marker rewritten or dropped
_MessageT, _fields_with_cache_control(blocks_rewritten, rewrite)
)
def _kept_within_budget(state: tuple[int, bool], marks: int) -> tuple[int, bool]:
remaining, _ = state
if marks > remaining:
return remaining, False
return remaining - marks, True
def _messages_within_cache_control_limit(messages: Sequence[_MessageT], reserved_marks: int) -> Sequence[_MessageT]:
budget: Final = max(MAX_CACHE_CONTROL_BLOCKS - reserved_marks, 0)
newest_first: Final = tuple(reversed(messages))
budget_after_each: Final = accumulate(
(_cache_control_count(message) for message in newest_first), _kept_within_budget, initial=(budget, True)
)
kept_flags: Final = tuple(kept for _, kept in islice(budget_after_each, 1, None))
kept_newest_first: Final = tuple(
message if kept else _with_cache_control(message, _dropped_cache_control)
for message, kept in zip(newest_first, kept_flags)
)
return tuple(reversed(kept_newest_first))
_SYSTEM_MESSAGE_FIELDS: Final = frozenset(("content", "cache_control"))
@ -799,7 +766,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT:
if fields is None:
return message
if fields.get("role") == "tool":
return _with_cache_control(message, _dropped_cache_control)
return with_cache_control(message, dropped_cache_control)
tool_calls: Final = fields.get("tool_calls")
if not isinstance(tool_calls, list):
return message
@ -808,7 +775,7 @@ def _without_tool_block_marks(message: _MessageT) -> _MessageT:
{
**fields,
"tool_calls": [
_block_with_cache_control(tool_call, _dropped_cache_control)
block_with_cache_control(tool_call, dropped_cache_control)
for tool_call in _OBJECT_LIST_ADAPTER.validate_python(tool_calls)
],
},
@ -822,7 +789,7 @@ class _ItemMarkRules:
external_marks: int
def sanitized(self, message: _MessageT) -> _MessageT:
valid: Final = _with_cache_control(message, _validated_cache_control)
valid: Final = with_cache_control(message, _validated_cache_control)
return _without_tool_block_marks(valid) if self.refuses_tool_block_marks else valid
def checked(self, message: _MessageT) -> _MessageT:
@ -845,15 +812,15 @@ def _request_messages_for_cache_points(messages: Sequence[_MessageT], rules: _It
as_system: Final = tuple(_developer_message_as_system_message(message) for message in messages)
if not rules.drop_params:
return as_system
return _messages_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks)
return newest_marks_within_cache_control_limit(tuple(map(rules.checked, as_system)), rules.external_marks)
def _replayed_messages_for_cache_points(
replayed_messages: Sequence[_MessageT], current_messages: Sequence[object], rules: _ItemMarkRules
) -> Sequence[_MessageT]:
as_system: Final = tuple(_developer_message_as_system_message(message) for message in replayed_messages)
current_marks: Final = sum(_cache_control_count(message) for message in current_messages)
return _messages_within_cache_control_limit(
current_marks: Final = sum(cache_control_count(message) for message in current_messages)
return whole_messages_within_cache_control_limit(
tuple(map(rules.sanitized, as_system)), current_marks + rules.external_marks
)
@ -1187,6 +1154,7 @@ class LiteLLMCompletionResponsesConfig:
responses_api_request=responses_api_request,
replay_reasoning=True,
item_cache_control=keeps_item_marks,
tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(request_kwargs),
native_compaction=native_compaction,
only_since_compaction=not native_compaction,
)
@ -1250,6 +1218,7 @@ class LiteLLMCompletionResponsesConfig:
responses_api_request: ResponsesAPIOptionalRequestParams | dict,
replay_reasoning: bool = False,
item_cache_control: bool = False,
tool_call_provider_fields: bool = False,
native_compaction: bool = False,
only_since_compaction: bool = False,
) -> list[
@ -1267,7 +1236,9 @@ class LiteLLMCompletionResponsesConfig:
assistant ``reasoning_content`` and signed ``thinking_blocks`` so the
provider gets its own chain-of-thought back instead of reading it as
visible text. ``item_cache_control`` belongs to the same callers, when
the target keeps an input item's ``cache_control`` on the message.
the target keeps an input item's ``cache_control`` on the message, and
``tool_call_provider_fields`` when the target reads a function call's
thought signature from ``provider_specific_fields`` instead of its id.
Callers that only inspect the messages (token counting, rate limiting,
guardrail scanning) leave both off, because they need every piece of text
@ -1305,7 +1276,7 @@ class LiteLLMCompletionResponsesConfig:
read_input: Final = _input_to_read(dumped_input, only_since_compaction, native_compaction)
messages.extend(
LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message(
input=read_input if item_cache_control else without_item_cache_control(read_input),
input=_without_item_keys(read_input, _dropped_item_keys(item_cache_control, tool_call_provider_fields)),
replay_reasoning=replay_reasoning,
native_compaction=native_compaction,
)
@ -1363,6 +1334,7 @@ class LiteLLMCompletionResponsesConfig:
ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id,
item_cache_control=keeps_item_marks,
client_response_ids=previous_response_client_ids,
tool_call_provider_fields=not embeds_thought_signature_in_tool_call_id(request_kwargs),
stored_turns=stored_turns,
)
chat_completion_session: Final = (
@ -1489,7 +1461,7 @@ class LiteLLMCompletionResponsesConfig:
messages.append(ChatCompletionUserMessage(role="user", content=input))
elif isinstance(input, list):
existing_tool_call_ids: Final[set[str]] = set()
for _input in input:
for _input in _without_paused_turn_thinking(input):
chat_completion_messages = (
LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message(
input_item=_input,
@ -2546,7 +2518,7 @@ class LiteLLMCompletionResponsesConfig:
tool_call: Final = (
ChatCompletionToolCallChunk(id=tool_call_id, type="function", function=tool_call_function, index=0)
if function_call_cache_control is None
else _MarkedToolCallChunk(
else _ReplayedToolCallChunk(
id=tool_call_id,
type="function",
function=tool_call_function,
@ -2554,9 +2526,17 @@ class LiteLLMCompletionResponsesConfig:
cache_control=function_call_cache_control,
)
)
provider_specific_fields: Final = as_str_mapping(function_call.get("provider_specific_fields"))
replayed_tool_call: Final = (
tool_call
if provider_specific_fields is None
else cast( # cast-ok: the same tool call plus the provider_specific_fields the client sent back
_ReplayedToolCallChunk, {**tool_call, "provider_specific_fields": provider_specific_fields}
)
)
# Create an assistant message with the tool call
return [ChatCompletionResponseMessage(tool_calls=[tool_call], role="assistant", content=None)]
return [ChatCompletionResponseMessage(tool_calls=[replayed_tool_call], role="assistant", content=None)]
@staticmethod
def _transform_input_file_item_to_file_item(item: Mapping[str, object]) -> _ResponsesFilePart:
@ -2601,45 +2581,55 @@ class LiteLLMCompletionResponsesConfig:
for item in content:
if isinstance(item, str):
content_list.append(item)
elif isinstance(item, dict):
if (
media := decode_responses_media(
fields := _validate_responses_media(_STR_KEY_DICT_ADAPTER, item)
elif (
isinstance(item, dict)
and (
part
:= LiteLLMCompletionResponsesConfig._transform_responses_api_content_part_to_chat_completion_part(
_validate_responses_media(_STR_KEY_DICT_ADAPTER, item)
)
) is not None:
content_list.append(dict(media))
elif fields.get("type") == "input_image":
content_list.append(dict(_responses_image_part(fields)))
elif fields.get("type") == "input_file":
content_list.append(dict(_responses_file_part(fields)))
elif fields.get("type") == "input_audio":
content_list.append(dict(_validate_responses_media(_AUDIO_PART_ADAPTER, fields)))
elif fields.get("type") == "encrypted_content":
encrypted_content = fields.get("encrypted_content")
if encrypted_content is not None:
content_list.append(
OpenAIChatCompletionTextObject(type="text", text=str(encrypted_content))
)
else:
# Skip text blocks with None text to avoid downstream errors
text_value = fields.get("text")
if text_value is None:
continue
content_block: dict[str, object] = {
"type": LiteLLMCompletionResponsesConfig._get_chat_completion_request_content_type(
content_type
if isinstance(content_type := fields.get("type"), str) and content_type
else "text"
),
"text": text_value,
}
if "cache_control" in fields:
content_block["cache_control"] = fields["cache_control"]
content_list.append(content_block)
)
is not None
):
content_list.append(part)
return content_list
else:
raise ValueError(f"Invalid content type: {type(content)}")
@staticmethod
def _transform_responses_api_content_part_to_chat_completion_part(
fields: dict[str, object],
) -> dict[str, object] | None:
if (media := decode_responses_media(fields)) is not None:
return dict(media)
part_type: Final = fields.get("type")
if part_type == "input_image":
return dict(_responses_image_part(fields))
if part_type == "input_file":
return dict(_responses_file_part(fields))
if part_type == "input_audio":
return dict(_validate_responses_media(_AUDIO_PART_ADAPTER, fields))
if part_type == "refusal":
refusal: Final = fields.get("refusal")
return dict(OpenAIChatCompletionTextObject(type="text", text=refusal)) if isinstance(refusal, str) else None
if part_type == "encrypted_content":
encrypted_content: Final = fields.get("encrypted_content")
return (
None
if encrypted_content is None
else dict(OpenAIChatCompletionTextObject(type="text", text=str(encrypted_content)))
)
text_value: Final = fields.get("text")
if text_value is None:
return None
return {
"type": LiteLLMCompletionResponsesConfig._get_chat_completion_request_content_type(
part_type if isinstance(part_type, str) and part_type else "text"
),
"text": text_value,
**({"cache_control": fields["cache_control"]} if "cache_control" in fields else {}),
}
@staticmethod
def _get_chat_completion_request_content_type(
content_type: str,
@ -3118,6 +3108,18 @@ class LiteLLMCompletionResponsesConfig:
# Default to completed for unknown finish reasons
return "completed"
@staticmethod
def _is_paused_turn(choice: Choices) -> bool:
return anthropic_stop_reason(choice.message) == "pause_turn"
@staticmethod
def _responses_status_for_choice(choice: Choices) -> ResponsesAPIStatus:
if LiteLLMCompletionResponsesConfig._is_paused_turn(choice):
return "completed"
return LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(
choice.finish_reason
)
@staticmethod
def _incomplete_details_for_finish_reason(
finish_reason: str | None,
@ -3254,19 +3256,21 @@ class LiteLLMCompletionResponsesConfig:
"""
if isinstance(chat_completion_response, dict):
chat_completion_response = ModelResponse(**chat_completion_response)
# Get finish_reason from the first choice to determine overall status
finish_reason: str | None = None
choices: Final[list[Choices]] = getattr(chat_completion_response, "choices", [])
if choices and len(choices) > 0:
finish_reason = choices[0].finish_reason
finish_reason: Final = choices[0].finish_reason if choices else None
paused_turn: Final = bool(choices) and LiteLLMCompletionResponsesConfig._is_paused_turn(choices[0])
echoed: Final = _echoable_request_params(responses_api_request)
chat_usage: Final = getattr(chat_completion_response, "usage", None)
incomplete_details: Final = LiteLLMCompletionResponsesConfig._incomplete_details_for_finish_reason(
finish_reason=finish_reason,
existing=getattr(chat_completion_response, "incomplete_details", None),
completion_tokens=getattr(chat_usage, "completion_tokens", None),
max_output_tokens=echoed.get("max_output_tokens"),
incomplete_details: Final = (
None
if paused_turn
else LiteLLMCompletionResponsesConfig._incomplete_details_for_finish_reason(
finish_reason=finish_reason,
existing=getattr(chat_completion_response, "incomplete_details", None),
completion_tokens=getattr(chat_usage, "completion_tokens", None),
max_output_tokens=echoed.get("max_output_tokens"),
)
)
responses_api_response: Final[ResponsesAPIResponse] = ResponsesAPIResponse(
@ -3293,8 +3297,8 @@ class LiteLLMCompletionResponsesConfig:
max_output_tokens=echoed.get("max_output_tokens"),
previous_response_id=echoed.get("previous_response_id"),
reasoning=echoed.get("reasoning"),
status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(
finish_reason
status=(
LiteLLMCompletionResponsesConfig._responses_status_for_choice(choices[0]) if choices else "incomplete"
),
text=echoed.get("text") or {},
truncation=echoed.get("truncation"),
@ -3440,9 +3444,7 @@ class LiteLLMCompletionResponsesConfig:
GenericResponseOutputItem(
type="reasoning",
id=mint_reasoning_item_id(),
status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(
choice.finish_reason
),
status=LiteLLMCompletionResponsesConfig._responses_status_for_choice(choice),
role="assistant",
content=[
OutputText(
@ -3565,24 +3567,34 @@ class LiteLLMCompletionResponsesConfig:
choice=choice,
)
message_output_items.extend(image_generation_items)
elif choice.message.content is not None or getattr(choice.message, "annotations", None):
elif content := LiteLLMCompletionResponsesConfig._message_output_content(choice.message):
message_output_items.append(
GenericResponseOutputItem(
type="message",
id=f"msg_{uuid.uuid4()}",
status=LiteLLMCompletionResponsesConfig._map_chat_completion_finish_reason_to_responses_status(
choice.finish_reason
),
status=LiteLLMCompletionResponsesConfig._responses_status_for_choice(choice),
role=choice.message.role,
content=[
LiteLLMCompletionResponsesConfig._transform_chat_message_to_response_output_text(
choice.message
)
],
content=list(content),
)
)
return message_output_items
@staticmethod
def _message_output_content(message: Message) -> tuple[OutputText | ResponseOutputRefusal, ...]:
refusal: Final = openai_chat_refusal_text(message)
refusal_parts: Final = (ResponseOutputRefusal(type="refusal", refusal=refusal),) if refusal is not None else ()
has_text_part: Final = (
bool(message.content)
or bool(getattr(message, "annotations", None))
or (message.content is not None and not refusal_parts)
)
text_parts: Final = (
(LiteLLMCompletionResponsesConfig._transform_chat_message_to_response_output_text(message),)
if has_text_part
else ()
)
return (*text_parts, *refusal_parts)
@staticmethod
def _transform_responses_api_outputs_to_chat_completion_messages(
responses_api_output: ResponsesAPIResponse,

View file

@ -23,6 +23,7 @@ from litellm.responses.mcp.tool_precheck import (
LITELLM_PROXY_MCP_SERVER_URL_PREFIX,
may_name_litellm_mcp_gateway,
names_gateway_explicitly,
names_whole_gateway,
proxy_path_mcp_name,
)
from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator
@ -182,8 +183,8 @@ class LiteLLM_Proxy_MCP_Handler:
@staticmethod
async def _apply_toolset_permissions(
resolved_toolset_ids: list[str],
resolved_mcp_servers: list[str],
resolved_toolset_ids: Sequence[str],
resolved_mcp_servers: Sequence[str],
user_api_key_auth: "UserAPIKeyAuth",
) -> "UserAPIKeyAuth":
"""Apply resolved toolset permissions to user_api_key_auth and return updated auth."""
@ -195,7 +196,7 @@ class LiteLLM_Proxy_MCP_Handler:
)
tool_permissions: Final = await global_mcp_server_manager.resolve_toolset_tool_permissions(
toolset_ids=resolved_toolset_ids
toolset_ids=list(resolved_toolset_ids)
)
all_server_ids: Final = list(set(tool_permissions.keys()) | set(resolved_mcp_servers))
existing_op: Final = user_api_key_auth.object_permission
@ -224,6 +225,62 @@ class LiteLLM_Proxy_MCP_Handler:
verbose_logger.debug("Could not apply toolset permissions: %s", _e)
return user_api_key_auth
@staticmethod
async def _resolve_proxy_mcp_name(
name: str,
user_api_key_auth: "UserAPIKeyAuth | None",
granted_toolsets: "GrantedToolsetIds | None",
) -> tuple[Literal["server", "toolset", "denied_toolset"], str]:
from litellm.proxy.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
if global_mcp_server_manager.get_mcp_server_by_name(name):
return ("server", name)
try:
from litellm.proxy.proxy_server import prisma_client
if prisma_client is None:
return ("server", name)
toolset: Final = await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, name)
if toolset is None:
return ("server", name)
from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import (
has_team_unbound_mcp_admin_view,
)
from litellm.proxy.mcp_server.ui_session_utils import (
granted_toolset_ids,
)
denied: Final = (
user_api_key_auth is not None
and not has_team_unbound_mcp_admin_view(user_api_key_auth)
and toolset.toolset_id not in await (granted_toolsets or granted_toolset_ids)(user_api_key_auth)
)
except Exception as _e:
verbose_logger.debug("Could not resolve '%s' as toolset: %s", name, _e)
return ("server", name)
if not denied:
return ("toolset", toolset.toolset_id)
if await LiteLLM_Proxy_MCP_Handler._holds_access_group_named(name, user_api_key_auth):
return ("server", name)
verbose_logger.debug("Key does not have access to toolset '%s', denying it.", name)
return ("denied_toolset", name)
@staticmethod
async def _holds_access_group_named(name: str, user_api_key_auth: "UserAPIKeyAuth | None") -> bool:
from litellm.proxy.mcp_server.auth.user_api_key_auth_mcp import (
MCPRequestHandler,
)
from litellm.proxy.mcp_server.mcp_server_manager import (
global_mcp_server_manager,
)
group_server_ids: Final = await MCPRequestHandler.get_mcp_servers_from_access_groups([name])
return bool(group_server_ids) and not frozenset(group_server_ids).isdisjoint(
await global_mcp_server_manager.get_allowed_mcp_servers(user_api_key_auth)
)
@staticmethod
@catalog_operation(global_manager)
async def _get_mcp_tools_from_manager(
@ -256,6 +313,7 @@ class LiteLLM_Proxy_MCP_Handler:
from litellm.proxy.mcp_server.server import (
_get_allowed_mcp_servers_from_mcp_server_names,
_get_tools_from_mcp_servers,
raise_generic_mcp_access_denial,
)
mcp_servers: Final = [
@ -268,36 +326,15 @@ class LiteLLM_Proxy_MCP_Handler:
# Resolve toolset names: collect all toolset IDs first, then apply their
# combined permissions in a single pass so multiple toolsets are unioned
# rather than the last one overwriting the others.
resolved_mcp_servers: Final[list[str]] = []
resolved_toolset_ids: Final[list[str]] = []
for name in mcp_servers:
if not global_mcp_server_manager.get_mcp_server_by_name(name):
try:
from litellm.proxy.proxy_server import prisma_client
if prisma_client is not None:
toolset = await global_mcp_server_manager.get_toolset_by_name_cached(prisma_client, name)
if toolset is not None:
if user_api_key_auth is not None:
from litellm.proxy.management_endpoints.common_utils import (
user_api_key_has_admin_view,
)
from litellm.proxy.mcp_server.ui_session_utils import (
granted_toolset_ids,
)
if not user_api_key_has_admin_view(user_api_key_auth) and toolset.toolset_id not in (
await (granted_toolsets or granted_toolset_ids)(user_api_key_auth)
):
verbose_logger.debug("Key does not have access to toolset '%s', skipping.", name)
continue
resolved_toolset_ids.append(toolset.toolset_id)
# Don't add to resolved_mcp_servers — toolset scope
# restricts via object_permission, not server name filter.
continue
except Exception as _e:
verbose_logger.debug("Could not resolve '%s' as toolset: %s", name, _e)
resolved_mcp_servers.append(name)
resolutions: Final = tuple(
[
await LiteLLM_Proxy_MCP_Handler._resolve_proxy_mcp_name(name, user_api_key_auth, granted_toolsets)
for name in mcp_servers
]
)
resolved_mcp_servers: Final = tuple(value for kind, value in resolutions if kind == "server")
resolved_toolset_ids: Final = tuple(value for kind, value in resolutions if kind == "toolset")
denied_toolset_names: Final = tuple(value for kind, value in resolutions if kind == "denied_toolset")
# Apply all resolved toolsets at once (union), avoiding permission overwrite.
if resolved_toolset_ids and user_api_key_auth is not None:
@ -311,7 +348,10 @@ class LiteLLM_Proxy_MCP_Handler:
# full union (toolset server IDs + direct server names). Passing a name-based
# filter here would exclude those toolset server IDs (which are UUIDs, not
# names), so use None and let the auth object's mcp_servers do the filtering.
effective_server_filter: Final = None if resolved_toolset_ids else (resolved_mcp_servers or None)
asks_whole_gateway: Final = any(map(names_whole_gateway, mcp_tools_with_litellm_proxy or ()))
if denied_toolset_names and not (asks_whole_gateway or resolved_mcp_servers or resolved_toolset_ids):
raise_generic_mcp_access_denial(denied_toolset_names)
effective_server_filter: Final = None if resolved_toolset_ids else (list(resolved_mcp_servers) or None)
listing: Final = await _get_tools_from_mcp_servers(
user_api_key_auth=user_api_key_auth,

View file

@ -22,6 +22,13 @@ def names_gateway_explicitly(tool: object) -> bool:
return (_mcp_server_url(tool) or "").startswith(LITELLM_PROXY_MCP_SERVER_URL)
def names_whole_gateway(tool: object) -> bool:
server_url: Final = _mcp_server_url(tool) or ""
return server_url.startswith(LITELLM_PROXY_MCP_SERVER_URL) and not server_url.startswith(
LITELLM_PROXY_MCP_SERVER_URL_PREFIX
)
def proxy_path_mcp_name(tool: object) -> str | None:
server_url: Final = _mcp_server_url(tool)
match: Final = None if server_url is None else _PROXY_MCP_PATH_RE.match(server_url)

View file

@ -35,9 +35,10 @@ import litellm
from litellm.constants import (
EMPTY_MAPPING,
LITELLM_MAX_STREAMING_DURATION_SECONDS,
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY,
STREAM_SSE_DONE_STRING,
)
from litellm.exceptions import MidStreamFallbackError, RateLimitError
from litellm.exceptions import MidStreamFallbackError, RateLimitError, ServiceUnavailableError
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import (
process_response_headers,
@ -100,6 +101,12 @@ class ProjectQuotaCallback(Protocol):
) -> None: ...
class TokenLimitCallback(Protocol):
async def enforce_token_limits_for_frame(
self, user_api_key_dict: UserAPIKeyAuth, requested_model: str | None
) -> None: ...
@lru_cache(maxsize=1)
def get_openai_response_types():
from litellm.types.llms import openai as openai_types
@ -2158,6 +2165,25 @@ class _RefusedTurn:
ResponsesWsTurnAdmission: TypeAlias = Callable[[], Awaitable[Mapping[str, object]]]
def with_token_limit_checks(
turn_admission: ResponsesWsTurnAdmission | None,
token_limit_callbacks: Sequence[TokenLimitCallback],
user_api_key_dict: UserAPIKeyAuth | None,
requested_model: str | None,
) -> ResponsesWsTurnAdmission | None:
if not token_limit_callbacks or user_api_key_dict is None:
return turn_admission
async def admit() -> Mapping[str, object]:
for callback in token_limit_callbacks:
await callback.enforce_token_limits_for_frame(
user_api_key_dict=user_api_key_dict, requested_model=requested_model
)
return MappingProxyType({}) if turn_admission is None else await turn_admission()
return admit
async def _admit_frame_turn(
turn_admission: ResponsesWsTurnAdmission | None, raw_message: str
) -> _AdmittedTurn | _RefusedTurn:
@ -2173,22 +2199,24 @@ async def _admit_frame_turn(
return _RefusedTurn(e)
def _turn_refusal_frame(refusal: RateLimitError) -> str:
def turn_refusal_frame(refusal: RateLimitError) -> str:
return json.dumps(
{"type": "error", "status": 429, "error": {"type": "rate_limit_exceeded", "message": str(refusal)}}
)
def _turn_admission_error_frame(failure: Exception) -> str:
return json.dumps({"type": "error", "status": 500, "error": {"type": "server_error", "message": str(failure)}})
def _turn_admission_error_frame(failure: Exception, status: int) -> str:
return json.dumps({"type": "error", "status": status, "error": {"type": "server_error", "message": str(failure)}})
def _turn_rejection_frame(rejection: Exception) -> str:
match rejection:
case RateLimitError():
return _turn_refusal_frame(rejection)
return turn_refusal_frame(rejection)
case ServiceUnavailableError():
return _turn_admission_error_frame(rejection, rejection.status_code)
case _:
return _turn_admission_error_frame(rejection)
return _turn_admission_error_frame(rejection, 500)
RESPONSES_WS_LOGGED_EVENT_TYPES: Final = [
@ -3015,7 +3043,7 @@ class ResponsesWebSocketStreaming:
async def _send_turn_refusal(self, refusal: RateLimitError) -> None:
try:
await self.websocket.send_text(_turn_refusal_frame(refusal))
await self.websocket.send_text(turn_refusal_frame(refusal))
except Exception: # noqa: BLE001, S110 # client may already be gone
pass
@ -3569,7 +3597,7 @@ class ManagedResponsesWebSocketHandler:
self.quota_callbacks, self.user_api_key_dict, self.model_group or self.model, raw_message
)
except RateLimitError as e:
await self._send_frame(_turn_refusal_frame(e))
await self._send_frame(turn_refusal_frame(e))
return
guarded: Final = await self._guard_frame(raw_message)
if guarded is None:
@ -3613,6 +3641,7 @@ class ManagedResponsesWebSocketHandler:
call_kwargs["litellm_metadata"] = {
**(turn_litellm_metadata if _is_json_object(turn_litellm_metadata) else {}),
**verdict.reservation,
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY: self.logging_obj.litellm_call_id,
}
try:

View file

@ -147,6 +147,7 @@ from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, reject_api_key_secret_reference
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.deployment_pricing import deployment_pricing_models
from litellm.litellm_core_utils.get_litellm_params import SUPPLEMENTAL_KWARGS_KEYS
from litellm.litellm_core_utils.get_llm_provider_logic import (
declared_authenticating_provider,
inferred_provider,
@ -483,6 +484,7 @@ from litellm.types.router import (
SearchToolTypedDict,
TaggedPreRoutingStrategy,
holds_secret_pointer,
validate_litellm_params_strictly,
)
from litellm.types.services import ServiceTypes
from litellm.types.utils import (
@ -640,6 +642,7 @@ _PRE_CALL_CHECK_FILTERED_PARAMS: Final = frozenset({"response_format"})
_STREAM_TIMEOUT_AWARE_GENERIC_HANDLERS: Final = frozenset({"anthropic_messages", "aresponses", "responses"})
_video_call_deployment_id: Final[ContextVar[str | None]] = ContextVar("video_call_deployment_id", default=None)
_routed_call_type: Final[ContextVar[str | None]] = ContextVar("routed_call_type", default=None)
_DECISIONS_MEDIA_CALL_TYPES: Final = frozenset({"adecisions", "decisions", "asystemone", "systemone"})
_ROUTE_CAPABILITY_RESOLVERS: Final[Mapping[str, Callable[[str, LlmProviders], object | None]]] = MappingProxyType(
{
"adecisions": ProviderConfigManager.get_provider_decisions_config,
@ -727,6 +730,35 @@ def _drops_malformed_request_tools(
return drop_policy({**deployment_params, **set_defaults, **request}).drops("tools")
def _malformed_request_params_the_deployment_sets(
deployment_params: Mapping[str, object], defaults: Mapping[str, object], request: Mapping[str, object]
) -> frozenset[str]:
overrides: Final = {
key: value
for key, value in request.items()
if key in SUPPLEMENTAL_KWARGS_KEYS and deployment_params.get(key) is not None
}
if not overrides:
return frozenset()
try:
validate_litellm_params_strictly(GenericLiteLLMParams, overrides)
except ValidationError as error:
set_defaults: Final = {name: value for name, value in defaults.items() if value is not None}
policy: Final = drop_policy({**deployment_params, **set_defaults, **request})
malformed: Final = (str(detail["loc"][0]) for detail in error.errors() if detail["loc"])
return frozenset(key for key in malformed if policy.drops(key))
return frozenset()
def _drop_malformed_request_params_the_deployment_sets(
kwargs: dict[str, object], # mutable-ok: drops the keys from the router's request kwargs in place
deployment_params: Mapping[str, object],
defaults: Mapping[str, object],
) -> None:
for key in _malformed_request_params_the_deployment_sets(deployment_params, defaults, kwargs):
kwargs.pop(key)
def _merged_tools(deployment_tools: Sequence[object], request_tools: object) -> list[object] | None:
if not isinstance(request_tools, (list, tuple)):
return None
@ -4162,13 +4194,16 @@ class Router(DeprecatedRouterMembers):
raise e.original_exception from e
raise e
def _active_default_litellm_params(self) -> Mapping[str, object]:
default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self._core_default_litellm_params)
return {key: value for key, value in default_params.items() if value is not None}
def _params_sent_to_deployment(
self, request_kwargs: Mapping[str, object], deployment: Deployment | None
) -> Mapping[str, object]:
"""The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies."""
deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {}
default_params: Final = _REQUEST_KWARGS_ADAPTER.validate_python(self._core_default_litellm_params)
defaults: Final = {key: value for key, value in default_params.items() if value is not None}
defaults: Final = self._active_default_litellm_params()
deployment_tool_choice: Final = (
{"tool_choice": deployment_params["tool_choice"]}
if deployment_params.get("tool_choice") is not None
@ -5223,6 +5258,11 @@ class Router(DeprecatedRouterMembers):
if isinstance(raw_deployment_params, LiteLLM_Params)
else raw_deployment_params
)
_drop_malformed_request_params_the_deployment_sets(
kwargs, # pyright: ignore[reportUnknownArgumentType] # kwargs is the router's untyped request dict
deployment_context,
defaults,
)
declared_provider: Final = deployment_context.get("custom_llm_provider")
declared_model: Final = deployment_context.get("model")
deployment_provider: Final = (
@ -10980,7 +11020,9 @@ class Router(DeprecatedRouterMembers):
merged_params: Final[Mapping[str, object]] = (
_litellm_params if zeroed_pricing is None else MappingProxyType({**_litellm_params, **zeroed_pricing})
)
litellm_params: Final[LiteLLM_Params] = LiteLLM_Params.model_validate(dict(**merged_params))
litellm_params: Final[LiteLLM_Params] = validate_litellm_params_strictly(
LiteLLM_Params, dict(**merged_params)
)
declared_params: Final = _LITELLM_PARAMS_ADAPTER.validate_python(_litellm_params)
warn_on_provider_credential_mismatch(model_name=_model_name, litellm_params=declared_params)
warn_on_ignored_model_capability_flags(model_name=_model_name, litellm_params=declared_params)
@ -14952,6 +14994,62 @@ class Router(DeprecatedRouterMembers):
return healthy_deployments
return cast(_RouteDeploymentsT, filtered_deployments) # cast-ok: same list shape as the input candidates
@staticmethod
def _decisions_deployment_refuses_media(
deployment: DeploymentTypedDict, group: str, request_kwargs: Mapping[str, object], *, audio: bool, videos: bool
) -> bool:
from litellm.decisions.call import deployment_refuses_media
plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts
model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict
deployment_params: Final = cast(Mapping[str, object], deployment["litellm_params"]) # cast-ok: plain dict
try:
model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group)
return deployment_refuses_media(
model,
custom_llm_provider,
model_info,
{**deployment_params, **request_kwargs},
audio=audio,
videos=videos,
)
except Exception as e: # noqa: BLE001 # best-effort filter: a deployment it cannot resolve answers for itself
verbose_router_logger.debug(
"litellm.router.py::_decisions_deployment_refuses_media: keeping a deployment of model=%s. Got - %s",
group,
e,
)
return False
def _without_decisions_deployments_refusing_media(
self, healthy_deployments: _RouteDeploymentsT, group: str, request_kwargs: Mapping[str, object] | None
) -> _RouteDeploymentsT:
call_kwargs: Final = request_kwargs or {}
audio: Final = bool(call_kwargs.get("audio"))
videos: Final = bool(call_kwargs.get("videos"))
if (
_routed_call_type.get() not in _DECISIONS_MEDIA_CALL_TYPES
or not (audio or videos)
or isinstance(healthy_deployments, Mapping)
):
return healthy_deployments
sent_kwargs: Final = {**self._active_default_litellm_params(), **call_kwargs}
deployments_taking_media: Final = [
deployment
for deployment in healthy_deployments
if not Router._decisions_deployment_refuses_media(
cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape
group,
sent_kwargs,
audio=audio,
videos=videos,
)
]
if not deployments_taking_media:
return healthy_deployments
return cast(_RouteDeploymentsT, deployments_taking_media) # cast-ok: same list shape as the input candidates
@staticmethod
def _without_declared_text_only_deployments(
healthy_deployments: _RouteDeploymentsT,
@ -15676,10 +15774,14 @@ class Router(DeprecatedRouterMembers):
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
_target_order: Final = (request_kwargs or {}).pop("_target_order", None)
batch_candidates: Final = self._without_declared_text_only_deployments(
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
batch_candidates: Final = self._without_decisions_deployments_refusing_media(
self._without_declared_text_only_deployments(
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
model,
messages,
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
),
model,
messages,
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
)
embedding_batch_admitting: Final = deployments_admitting_embedding_batch(
@ -16932,10 +17034,14 @@ class Router(DeprecatedRouterMembers):
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
_target_order: Final = (request_kwargs or {}).pop("_target_order", None)
batch_candidates: Final = self._without_declared_text_only_deployments(
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
batch_candidates: Final = self._without_decisions_deployments_refusing_media(
self._without_declared_text_only_deployments(
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
model,
messages,
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
),
model,
messages,
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
)
embedding_batch_admitting: Final = deployments_admitting_embedding_batch(

View file

@ -288,6 +288,7 @@ class PromptOptions:
@dataclass(frozen=True, slots=True, kw_only=True)
class ResponseOptions:
merge_reasoning_content_in_choices: bool | None = None
thought_signature_in_tool_call_id: bool | None = None
enable_json_schema_validation: bool | None = None
complete_response: bool | None = None
keepalive_seconds: float | None = None

View file

@ -751,7 +751,9 @@ class AnthropicOutputTokensDetails(LiteLLMBaseModel):
thinking_tokens: int | None = None
AnthropicFinishReason = Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction"]
AnthropicFinishReason = Literal[
"end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "pause_turn", "compaction"
]
class AnthropicResponse(LiteLLMBaseModel):

View file

@ -98,7 +98,7 @@ class AnthropicMessagesResponse(TypedDict, total=False):
model: str | None # This represents the Model type from Anthropic
role: Literal["assistant"] | None
stop_reason: ReadOnly[
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction"] | None
Literal["end_turn", "max_tokens", "stop_sequence", "tool_use", "refusal", "compaction", "pause_turn"] | None
]
stop_details: NotRequired[ReadOnly[AnthropicStopDetails | None]]
stop_sequence: str | None

View file

@ -70,6 +70,7 @@ from pydantic import (
FieldSerializationInfo,
NonNegativeInt,
PrivateAttr,
SerializeAsAny,
SerializerFunctionWrapHandler,
field_serializer,
field_validator,
@ -402,7 +403,7 @@ class OpenAIFileObject(LiteLLMBaseModel):
filename: str
"""The name of the file."""
object: Literal["file"]
object: Literal["file"] = "file"
"""The object type, which is always `file`."""
purpose: OpenAIFilesPurpose
@ -440,6 +441,11 @@ class OpenAIFileObject(LiteLLMBaseModel):
_hidden_params: dict = PrivateAttr(default={"response_cost": 0.0}) # no cost for writing a file
@field_validator("object", mode="before")
@classmethod
def _default_missing_object_type(cls, value: _JsonValue) -> _JsonValue:
return "file" if value is None else value
@property
def hidden_params(self) -> dict[str, builtins.object]: # mutable-ok: API requires mutation
return self._hidden_params
@ -1850,7 +1856,7 @@ class ContentPartAddedEvent(BaseLiteLLMOpenAIResponseObject):
item_id: str
output_index: int
content_index: int
part: BaseLiteLLMOpenAIResponseObject
part: SerializeAsAny[BaseLiteLLMOpenAIResponseObject]
class ContentPartDonePartOutputText(BaseLiteLLMOpenAIResponseObject):

View file

@ -1,5 +1,6 @@
from dataclasses import dataclass
from enum import Enum
from typing import Final
from typing import Final, Protocol
from pydantic import Field
from typing_extensions import TypedDict
@ -75,6 +76,19 @@ class PassthroughStandardLoggingPayload(TypedDict, total=False):
"""
@dataclass(frozen=True, slots=True)
class WebSocketSessionUsage:
model: str
custom_llm_provider: str
response_cost: float | None
class WebSocketSessionMeter(Protocol):
def measure(self, client_text: str) -> int: ...
def price(self, total: int) -> WebSocketSessionUsage: ...
class PassThroughGuardrailSettings(LiteLLMPydanticObjectBase):
"""
Settings for a specific guardrail on a passthrough endpoint.

View file

@ -4,6 +4,7 @@ from typing import Final, Literal, Optional, Union
from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
from openai.types.responses.response_function_web_search import ActionSearchSource, ResponseFunctionWebSearch
from openai.types.responses.response_output_refusal import ResponseOutputRefusal
from pydantic import ConfigDict, PrivateAttr
from typing_extensions import TypedDict
@ -154,7 +155,7 @@ class GenericResponseOutputItem(BaseLiteLLMOpenAIResponseObject):
id: str
status: str # "completed", "in_progress", etc.
role: str # "assistant", "user", etc.
content: list[OutputText]
content: list[OutputText | ResponseOutputRefusal]
phase: Phase = None

View file

@ -25,7 +25,20 @@ from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
import httpx
from openai import Timeout as SDKTimeout
from pydantic import BeforeValidator, ConfigDict, Field, JsonValue, StrictInt, field_validator, model_validator
from pydantic import (
BaseModel,
BeforeValidator,
ConfigDict,
Field,
JsonValue,
StrictInt,
ValidationError,
ValidationInfo,
ValidatorFunctionWrapHandler,
WrapValidator,
field_validator,
model_validator,
)
from typing_extensions import NotRequired, Protocol, ReadOnly, Required, TypedDict, runtime_checkable
from litellm._logging import verbose_logger
@ -531,6 +544,27 @@ def holds_secret_pointer(param_name: str) -> bool:
_RESERVED_INIT_KEYS: Final = frozenset({"self", "params", "__class__"})
_STRICT_LITELLM_PARAMS: Final = MappingProxyType({"strict_litellm_params": True})
_StrictlyValidatedParams: Final = TypeVar("_StrictlyValidatedParams", bound=BaseModel)
def validate_litellm_params_strictly(
params_model: type[_StrictlyValidatedParams], params: Mapping[str, object]
) -> _StrictlyValidatedParams:
return params_model.model_validate(params, context=_STRICT_LITELLM_PARAMS)
def _none_when_malformed_outside_strict_validation(
value: object, handler: ValidatorFunctionWrapHandler, info: ValidationInfo
) -> object:
try:
return handler(value)
except ValidationError:
if info.context is _STRICT_LITELLM_PARAMS:
raise
return None
_DAILY_RESET_TIME_PATTERN: Final = re.compile(r"\d{2}:\d{2}")
@ -612,6 +646,9 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
)
model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True)
merge_reasoning_content_in_choices: bool | None = False
thought_signature_in_tool_call_id: Annotated[
bool | None, WrapValidator(_none_when_malformed_outside_strict_validation)
] = None
model_info: dict | None = None
mock_response: str | ModelResponse | Exception | object | None = None

View file

@ -106,6 +106,7 @@ from .llms.openai import (
OpenAIRealtimeStreamList,
ResponsesAPIResponse,
WebSearchOptions,
_JsonValue,
)
from .rerank import RerankResponse as RerankResponse
@ -4682,6 +4683,7 @@ class LlmProviders(str, Enum):
PRISM = "prism"
OPENINFER = "openinfer"
INFERENCEAPIS = "inferenceapis"
HOSTINGER_ROUTER = "hostinger_router"
ZEROGPU = "zerogpu"
DARKBLOOM = "darkbloom"
META = "meta"
@ -4963,29 +4965,50 @@ def _batch_usage_from_chat_usage(usage: Usage) -> BatchUsage:
)
OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS: Final = MappingProxyType(
{"pending": "validating", "running": "in_progress"}
)
def openai_batch_status(status: _JsonValue) -> _JsonValue:
return OPENAI_BATCH_STATUS_FOR_VLLM_ROUTER_STATUS.get(status, status) if isinstance(status, str) else status
class LiteLLMBatch(Batch):
_hidden_params: dict = {}
object: Literal["batch"] = "batch"
@field_validator("object", mode="before")
@classmethod
def _default_missing_object_type(cls, value: _JsonValue) -> _JsonValue:
return "batch" if value is None else value
@field_validator("status", mode="before")
@classmethod
def _map_vllm_router_status(cls, value: _JsonValue) -> _JsonValue:
return openai_batch_status(value)
@property
def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation
def hidden_params(self) -> dict[str, _JsonValue]: # mutable-ok: API requires mutation
return self._hidden_params
@hidden_params.setter
def hidden_params(self, hidden_params: dict[str, object]) -> None: # mutable-ok: API requires mutation
def hidden_params(self, hidden_params: dict[str, _JsonValue]) -> None: # mutable-ok: API requires mutation
self._hidden_params = hidden_params
usage: Usage | None = None
@field_validator("usage", mode="before")
@classmethod
def _usage_from_batch_token_counts(cls, value: object) -> object:
def _usage_from_batch_token_counts(cls, value: _JsonValue) -> _JsonValue:
counts: Final = _parse_batch_token_counts(value)
return value if counts is None else _usage_with_batch_token_counts(Usage.model_validate(value), counts)
@field_serializer("usage")
def _serialize_usage_with_batch_fields(
self, usage: Usage | None, info: FieldSerializationInfo
) -> Mapping[str, object] | None:
) -> Mapping[str, _JsonValue] | None:
if not isinstance(usage, Usage):
return usage
chat_fields: Final = usage.model_dump(

View file

@ -10159,7 +10159,7 @@ class ProviderConfigManager:
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
json_provider: Final = JSONProviderRegistry.get(provider.value)
if json_provider is not None and "/v1/messages" in json_provider.supported_endpoints:
if json_provider is not None and JSONProviderRegistry.serves_endpoint(provider.value, model, "/v1/messages"):
from litellm.llms.openai_like.messages.transformation import (
JSONProviderAnthropicMessagesConfig,
)
@ -10298,7 +10298,7 @@ class ProviderConfigManager:
return result
# Fall back to JSON providers (generic OpenAI-compatible)
if JSONProviderRegistry.exists(provider_str) and JSONProviderRegistry.supports_responses_api(provider_str):
if JSONProviderRegistry.serves_endpoint(provider_str, model, "/v1/responses"):
provider_config: Final = JSONProviderRegistry.get(provider_str)
if provider_config is not None:
return create_responses_config_class(provider_config)()

View file

@ -39687,6 +39687,57 @@
"/v1/images/generations"
]
},
"hostinger_router/gpt-5.2": {
"cache_read_input_token_cost": 0.00000021,
"input_cost_per_token": 0.0000021,
"litellm_provider": "hostinger_router",
"max_input_tokens": 272000,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 0.0000168,
"supports_function_calling": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_response_schema": true,
"supports_vision": true,
"supported_endpoints": ["/v1/chat/completions", "/v1/responses"],
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/claude-sonnet-5": {
"cache_creation_input_token_cost": 3.3e-06,
"cache_creation_input_token_cost_above_1hr": 5.28e-06,
"cache_read_input_token_cost": 2.64e-07,
"input_cost_per_token": 2.64e-06,
"litellm_provider": "hostinger_router",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 1.32e-05,
"supports_function_calling": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_response_schema": true,
"supported_endpoints": ["/v1/chat/completions", "/v1/messages"],
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/text-embedding-3-large": {
"input_cost_per_token": 1.716e-07,
"litellm_provider": "hostinger_router",
"max_input_tokens": 8191,
"mode": "embedding",
"output_cost_per_token": 0.0,
"source": "https://router.hostinger.com/v1/models"
},
"hostinger_router/text-embedding-3-small": {
"input_cost_per_token": 3e-08,
"litellm_provider": "hostinger_router",
"max_input_tokens": 8191,
"mode": "embedding",
"output_cost_per_token": 0.0,
"source": "https://router.hostinger.com/v1/models"
},
"hyperbolic/NousResearch/Hermes-3-Llama-3.1-70B": {
"input_cost_per_token": 1.2e-07,
"litellm_provider": "hyperbolic",
@ -51387,6 +51438,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 5e-06,
"output_cost_per_token_batches": 2.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -51415,6 +51471,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 5e-06,
"output_cost_per_token_batches": 2.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -52161,6 +52222,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_batches": 7.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,
@ -52266,6 +52332,11 @@
"max_input_tokens": 200000,
"max_output_tokens": 64000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"output_cost_per_token": 1.5e-05,
"output_cost_per_token_batches": 7.5e-06,
"regional_endpoint_uplift_multiplier": 1.1,

View file

@ -1087,6 +1087,32 @@
"generateContent": false
}
},
"hostinger_router": {
"display_name": "Hostinger Router (`hostinger_router`)",
"url": "https://docs.litellm.ai/docs/providers/hostinger_router",
"endpoints": {
"chat_completions": true,
"responses": true,
"text_completion": true,
"embeddings": true,
"image_generations": false,
"image_edits": false,
"audio_transcriptions": false,
"audio_speech": false,
"rerank": false,
"ocr": false,
"search": false,
"moderations": false,
"realtime": false,
"video_generations": false,
"batches": false,
"messages": true,
"interactions": true,
"systemone": false,
"a2a": true,
"generateContent": true
}
},
"crusoe": {
"display_name": "Crusoe (`crusoe`)",
"url": "https://docs.litellm.ai/docs/providers/crusoe",

View file

@ -383,6 +383,8 @@ exclude-newer = "3 days"
[tool.uv.exclude-newer-package]
# TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window
brotli = "2026-10-22T14:01:00Z"
# TODO: remove after 2026-10-31, once pypdf 6.21.4 clears the 3-day window
pypdf = "2026-10-28T17:01:00Z"
[tool.uv.sources]
litellm-proxy-extras = { workspace = true }

View file

@ -124,6 +124,8 @@ ignored_function_names = [
"_is_fallback_hop", # Tested through the order fallback hop tests in test_router_order_fallback.py
"_deployment_that_just_failed", # Tested through the same-boundary hop test in test_router_order_fallback.py
"_merged_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py
"_malformed_request_params_the_deployment_sets", # Run by _update_kwargs_with_deployment, tested through the drop_params deployment test in test_thought_signature_in_tool_call_id.py
"_drop_malformed_request_params_the_deployment_sets", # Run by _update_kwargs_with_deployment, tested through the drop_params deployment test in test_thought_signature_in_tool_call_id.py
"_drops_malformed_request_tools", # Tested through the _update_kwargs_with_deployment tools merge tests in test_router.py
"_collect_ptu_share_teams", # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py
"_resolve_team_ptu_ceiling", # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py
@ -185,6 +187,9 @@ ignored_function_names = [
"_aresponses_background_stream_can_release", # Tested through the background Responses stream tests in test_router.py
"_raise_released_stream_error", # Tested through the background Responses stream content-policy tests in test_router.py
"_mid_stream_error_before_output", # Tested through the Responses mid-stream fallback and background stream tests
"_without_decisions_deployments_refusing_media", # Tested through the mixed media-flag group tests in tests/unit/decisions/test_main.py
"_decisions_deployment_refuses_media", # Read by _without_decisions_deployments_refusing_media, tested through the same group tests
"_active_default_litellm_params", # Tested through the router default drop_params test in tests/unit/decisions/test_main.py
]

View file

@ -45,6 +45,9 @@
- {id: llm.chat_completions.vertex.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: vertex, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "P0 route; Vertex AI"}
- {id: llm.chat_completions.gemini.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini OpenAI-compatible chat translation"}
- {id: llm.chat_completions.gemini.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: gemini, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "test_chat_completions_regression_e2e.py", rationale: "Gemini chat cost lands in SpendLogs"}
- {id: llm.chat_completions.gemini.tool_use.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: tool_use, streaming: nonstream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "Gemini tool-call ids fit the AgentCore tool-use id shape"}
- {id: llm.chat_completions.gemini.tool_use.stream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: tool_use, streaming: stream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "Streamed Gemini tool-call ids fit the AgentCore tool-use id shape"}
- {id: llm.chat_completions.gemini.multi_turn.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: gemini, capability: multi_turn, streaming: nonstream, assertions: [works], source: "llm_translation/test_gemini_tool_call_id_length_e2e.py", rationale: "A short Gemini tool-call id still round-trips when the client drops provider fields"}
- {id: llm.chat_completions.hosted_vllm.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "OpenAI-compatible hosted_vllm chat is a confirmed self-hosted backend path"}
- {id: llm.chat_completions.cohere.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: cohere, capability: basic, streaming: nonstream, assertions: [works], source: "test_chat_completions_regression_e2e.py", rationale: "Cohere chat via OpenAI-compatible /chat/completions"}

View file

@ -0,0 +1,187 @@
from __future__ import annotations
import re
from collections.abc import Iterator
from typing import Final
import pytest
from e2e_config import unique_marker
from e2e_metadata import Capability, Domain, Mode, Provider, Route, Subject, meta
from lifecycle import ResourceManager
from llm_translation.sdk_clients import NO_PROXY_CACHE, SdkClients
from models import LiteLLMParamsBody
from openai.types.chat import (
ChatCompletion,
ChatCompletionAssistantMessageParam,
ChatCompletionChunk,
ChatCompletionMessageFunctionToolCallParam,
ChatCompletionToolMessageParam,
ChatCompletionToolParam,
ChatCompletionUserMessageParam,
)
from openai.types.chat.chat_completion_chunk import ChoiceDeltaToolCall
from openai.types.chat.chat_completion_message_function_tool_call import ChatCompletionMessageFunctionToolCall
from passthrough_client import PassthroughClient
pytestmark = pytest.mark.e2e
BACKEND_MODEL: Final = "gemini/gemini-3.8-flash"
GEMINI_API_KEY: Final = "os.environ/GEMINI_API_KEY"
# bedrock-agentcore HarnessToolUseId shape in botocore 1.43.6, read 2026-10-20
AGENTCORE_TOOL_USE_ID: Final = re.compile(r"[a-zA-Z0-9_-]{1,64}")
WEATHER_TOOL: Final[ChatCompletionToolParam] = {
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the weather for a city",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
},
}
def _subject(mode: Mode) -> Subject:
return Subject(
domain=Domain.LLM_TRANSLATION,
route=Route.CHAT_COMPLETIONS,
providers=(Provider.GEMINI,),
models=(BACKEND_MODEL,),
mode=mode,
capabilities=(Capability.FUNCTION_CALLING, Capability.TOOL_CHOICE),
)
def _register_model(client: PassthroughClient, resources: ResourceManager) -> str:
model_name: Final = f"e2e-gemini-tool-call-id-{unique_marker()}"
model_id: Final = client.proxy.create_model(
model_name,
LiteLLMParamsBody(
model=BACKEND_MODEL,
api_key=GEMINI_API_KEY,
thought_signature_in_tool_call_id=False,
),
)
resources.defer(lambda: client.proxy.delete_model(model_id))
return model_name
def _user_message() -> ChatCompletionUserMessageParam:
return {"role": "user", "content": f"Use get_weather for San Francisco and return its result. {unique_marker()}"}
def _tool_calls(completion: ChatCompletion) -> tuple[ChatCompletionMessageFunctionToolCall, ...]:
assert completion.choices, f"Gemini returned no choices: {completion.model_dump_json()[:400]}"
calls: Final = tuple(
call
for call in completion.choices[0].message.tool_calls or ()
if isinstance(call, ChatCompletionMessageFunctionToolCall)
)
assert calls, f"Gemini returned no tool calls for tool_choice='required': {completion.model_dump_json()[:400]}"
return calls
def _assert_agentcore_accepts_ids(ids: tuple[str, ...]) -> None:
rejected: Final = tuple(call_id for call_id in ids if not AGENTCORE_TOOL_USE_ID.fullmatch(call_id))
assert not rejected, f"tool call id(s) AgentCore would reject: {tuple(call_id[:80] for call_id in rejected)}"
def _streamed_tool_calls(chunks: tuple[ChatCompletionChunk, ...]) -> Iterator[ChoiceDeltaToolCall]:
for chunk in chunks:
for choice in chunk.choices:
yield from choice.delta.tool_calls or ()
def _short_id_tool_call(call: ChatCompletionMessageFunctionToolCall) -> ChatCompletionMessageFunctionToolCallParam:
return {
"id": call.id,
"type": "function",
"function": {"name": call.function.name, "arguments": call.function.arguments},
}
def _tool_result(call: ChatCompletionMessageFunctionToolCall) -> ChatCompletionToolMessageParam:
return {"role": "tool", "tool_call_id": call.id, "content": '{"temperature": 21, "condition": "sunny"}'}
class TestGeminiToolCallIdLength:
@pytest.mark.covers(
"llm.chat_completions.gemini.tool_use.nonstream.works",
exercised_on=["chat_completions"],
)
@meta(_subject(Mode.NONSTREAM))
def test_non_streaming_tool_call_id_fits_agentcore(
self, client: PassthroughClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model: Final = _register_model(client, resources)
completion: Final = sdk.openai(resources.key()).chat.completions.create(
model=model,
messages=[_user_message()],
tools=[WEATHER_TOOL],
tool_choice="required",
max_tokens=2048,
extra_body=NO_PROXY_CACHE,
)
_assert_agentcore_accepts_ids(tuple(call.id for call in _tool_calls(completion)))
@pytest.mark.covers(
"llm.chat_completions.gemini.tool_use.stream.works",
exercised_on=["chat_completions"],
)
@meta(_subject(Mode.STREAM))
def test_streaming_tool_call_id_fits_agentcore(
self, client: PassthroughClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model: Final = _register_model(client, resources)
chunks: Final[tuple[ChatCompletionChunk, ...]] = tuple(
sdk.openai(resources.key()).chat.completions.create(
model=model,
messages=[_user_message()],
tools=[WEATHER_TOOL],
tool_choice="required",
max_tokens=2048,
stream=True,
extra_body=NO_PROXY_CACHE,
)
)
ids: Final = tuple(call.id for call in _streamed_tool_calls(chunks) if call.id)
assert ids, f"Gemini stream returned no tool call ids in {len(chunks)} chunks"
_assert_agentcore_accepts_ids(ids)
@pytest.mark.covers(
"llm.chat_completions.gemini.multi_turn.nonstream.works",
exercised_on=["chat_completions"],
)
@meta(_subject(Mode.NONSTREAM))
def test_tool_call_round_trip_succeeds(
self, client: PassthroughClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model: Final = _register_model(client, resources)
openai_client: Final = sdk.openai(resources.key())
question: Final = _user_message()
first: Final = openai_client.chat.completions.create(
model=model,
messages=[question],
tools=[WEATHER_TOOL],
tool_choice="required",
max_tokens=2048,
extra_body=NO_PROXY_CACHE,
)
calls: Final = _tool_calls(first)
assistant: Final[ChatCompletionAssistantMessageParam] = {
"role": "assistant",
"tool_calls": [_short_id_tool_call(call) for call in calls],
}
second: Final = openai_client.chat.completions.create(
model=model,
messages=[question, assistant, *(_tool_result(call) for call in calls)],
tools=[WEATHER_TOOL],
tool_choice="none",
max_tokens=2048,
extra_body=NO_PROXY_CACHE,
)
assert second.choices, f"Gemini turn 2 returned no choices: {second.model_dump_json()[:400]}"
answer: Final = second.choices[0].message.content
assert answer and answer.strip(), f"Gemini turn 2 returned an empty answer: {second.model_dump_json()[:400]}"

View file

@ -1363,6 +1363,7 @@ class LiteLLMParamsBody(BaseModel):
realtime_protocol: str | None = None
allowed_openai_params: list[str] | None = None
drop_params: bool | None = None
thought_signature_in_tool_call_id: bool | None = None
aws_access_key_id: str | None = Field(default=None, repr=False)
aws_secret_access_key: str | None = Field(default=None, repr=False)
aws_region_name: str | None = None

View file

@ -3,7 +3,7 @@
PARITY pairs the streamed case with its plain twin where both bill the same row."""
import json
from typing import Final, cast
from typing import Final, Literal, cast
from pydantic import JsonValue
@ -41,6 +41,7 @@ from integration.cost_calculation.cases.anthropic import (
from integration.cost_calculation.bridge_cost_parity.case import BridgeCostParityTestCase, bridge_pairs
from integration.cost_calculation.cost_tracking_case import (
CostTrackingTestCase,
Deployment,
ExactExpected,
JsonResponse,
RecountExpected,
@ -3387,6 +3388,232 @@ VERTEX_AI_CLAUDE_HAIKU_5_5_BRIDGE_PAIRS: Final = bridge_pairs(
)
VERTEX_AI_CLAUDE_DEPLOYMENT_RATES: Final = Deployment(
input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.01
)
def _vertex_claude_web_search_message(web_search_requests: int) -> JsonResponse:
return JsonResponse(
content_type="application/json",
body={
"id": "msg_$REQUEST_ID",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": [{"type": "text", "text": "scripted answer for cost fixtures"}],
"stop_reason": "end_turn",
"usage": {
"input_tokens": 1840,
"output_tokens": 412,
"server_tool_use": {"web_search_requests": web_search_requests},
},
},
)
def _vertex_claude_web_search_stream(web_search_requests: int) -> SseResponse:
return SseResponse(
content_type="text/event-stream",
frames=sse_frames(
{
"type": "message_start",
"message": {
"id": "msg_$REQUEST_ID",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-5-5",
"content": [],
"stop_reason": None,
"usage": {"input_tokens": 1840},
},
},
{"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
{
"type": "content_block_delta",
"index": 0,
"delta": {"type": "text_delta", "text": "scripted answer for cost fixtures"},
},
{"type": "content_block_stop", "index": 0},
{
"type": "message_delta",
"delta": {"stop_reason": "end_turn"},
"usage": {"output_tokens": 412, "server_tool_use": {"web_search_requests": web_search_requests}},
},
{"type": "message_stop"},
),
)
def _vertex_claude_deployment_web_search_case(
name: str,
endpoint: Literal["/v1/chat/completions", "/v1/responses", "/v1/messages"],
request: dict[str, JsonValue],
response: JsonResponse | SseResponse,
tool_usage_cost: float,
deployment: Deployment = VERTEX_AI_CLAUDE_DEPLOYMENT_RATES,
) -> CostTrackingTestCase:
return CostTrackingTestCase(
name=name,
covers="quota_management.spend_tracking.cost_matrix.logs_cost",
model="vertex_ai/claude-sonnet-5-5",
endpoint=endpoint,
deployment=deployment,
request={"model": "$MODEL", **request},
response=response,
expected=ExactExpected(
spend=round(0.0039 + tool_usage_cost, 10),
input_cost=0.00184,
output_cost=0.00206,
prompt_tokens=1840,
completion_tokens=412,
tool_usage_cost=tool_usage_cost,
),
)
VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL: Final[dict[str, JsonValue]] = {
"type": "web_search_20250305",
"name": "web_search",
"max_uses": 3,
}
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_rates_messages_web_search_two_requests",
"/v1/messages",
{
"max_tokens": 412,
"messages": [{"role": "user", "content": "vertex claude deployment rates messages web search"}],
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
},
_vertex_claude_web_search_message(2),
0.02,
)
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM: Final = _vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_rates_messages_web_search_two_requests_stream",
"/v1/messages",
{
"max_tokens": 412,
"messages": [{"role": "user", "content": "vertex claude deployment rates messages web search stream"}],
"stream": True,
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
},
_vertex_claude_web_search_stream(2),
0.02,
)
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE: Final = (
_vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_zero_search_rate_messages_two_requests",
"/v1/messages",
{
"max_tokens": 412,
"messages": [{"role": "user", "content": "vertex claude deployment zero search rate messages"}],
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
},
_vertex_claude_web_search_message(2),
0.0,
Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.0),
)
)
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY: Final = (
_vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_token_rates_only_messages_web_search_two_requests",
"/v1/messages",
{
"max_tokens": 412,
"messages": [{"role": "user", "content": "vertex claude deployment token rates only messages web search"}],
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
},
_vertex_claude_web_search_message(2),
0.034,
Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06),
)
)
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_three_requests",
"/v1/chat/completions",
{
"messages": [{"role": "user", "content": "vertex claude deployment rates chat web search"}],
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
"allowed_openai_params": ["web_search_options"],
},
_vertex_claude_web_search_message(3),
0.03,
)
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM: Final = _vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_rates_chat_web_search_two_requests_stream",
"/v1/chat/completions",
{
"messages": [{"role": "user", "content": "vertex claude deployment rates chat web search stream"}],
"stream": True,
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
"allowed_openai_params": ["web_search_options"],
},
_vertex_claude_web_search_stream(2),
0.02,
)
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH: Final = _vertex_claude_deployment_web_search_case(
"vertex_ai/claude-sonnet-5-5-deployment_rates_responses_web_search_two_requests",
"/v1/responses",
{"input": "vertex claude deployment rates responses web search", "tools": [{"type": "web_search"}]},
_vertex_claude_web_search_message(2),
0.02,
)
GEMINI_3_1_PRO_PREVIEW_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM: Final = CostTrackingTestCase(
name="vertex_ai/gemini-3.1-pro-preview-deployment_rates_messages_web_search_two_queries_stream",
covers="quota_management.spend_tracking.scripted_wire.logs_cost",
model="vertex_ai/gemini-3.1-pro-preview",
endpoint="/v1/messages",
deployment=Deployment(input_cost_per_token=1e-06, output_cost_per_token=5e-06, search_cost_per_query=0.03),
request={
"model": "$MODEL",
"max_tokens": 412,
"messages": [{"role": "user", "content": "vertex gemini deployment rates messages web search stream"}],
"stream": True,
"tools": [VERTEX_AI_CLAUDE_WEB_SEARCH_TOOL],
},
response=SseResponse(
content_type="text/event-stream",
frames=sse_frames(
{
"candidates": [
{
"content": {"parts": [{"text": "scripted answer for cost fixtures"}], "role": "model"},
"index": 0,
}
],
"modelVersion": "gemini-3.1-pro-preview",
},
{
"candidates": [
{
"content": {"parts": [{"text": ""}], "role": "model"},
"finishReason": "STOP",
"index": 0,
"groundingMetadata": {"webSearchQueries": ["query 0", "query 1"]},
}
],
"usageMetadata": {
"promptTokenCount": 1840,
"candidatesTokenCount": 412,
"totalTokenCount": 2252,
"promptTokensDetails": [{"modality": "TEXT", "tokenCount": 1840}],
},
"modelVersion": "gemini-3.1-pro-preview",
},
),
),
expected=ExactExpected(
spend=0.0339,
input_cost=0.00184,
output_cost=0.00206,
prompt_tokens=1840,
completion_tokens=412,
tool_usage_cost=0.03,
),
)
GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT: Final = _gemini_case(
GEMINI_3_8_FLASH_INPUT_TEXT, "vertex_ai/gemini-3.1-pro-preview-input_text", "vertex_ai/gemini-3.1-pro-preview",
ExactExpected(spend=0.0094864, input_cost=0.004048, output_cost=0.0054384, prompt_tokens=1840, completion_tokens=412),
@ -3725,6 +3952,14 @@ CASES: Final[tuple[CostTrackingTestCase, ...]] = (
VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_INPUT_TEXT,
VERTEX_AI_CLAUDE_SONNET_5_5_MESSAGES_STREAM,
*(pair.bridged for pair in VERTEX_AI_CLAUDE_SONNET_5_5_BRIDGE_PAIRS),
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_ZERO_RATE,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_MESSAGES_WEB_SEARCH_TOKEN_RATES_ONLY,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_CHAT_WEB_SEARCH_STREAM,
VERTEX_AI_CLAUDE_SONNET_5_5_DEPLOYMENT_RESPONSES_WEB_SEARCH,
GEMINI_3_1_PRO_PREVIEW_DEPLOYMENT_MESSAGES_WEB_SEARCH_STREAM,
GEMINI_3_1_PRO_PREVIEW_INPUT_TEXT,
GEMINI_3_1_PRO_PREVIEW_STREAM,
GEMINI_3_1_PRO_PREVIEW_CACHE_READ,

View file

@ -9,6 +9,15 @@ from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue
MODEL: Final = "claude-haiku-4-5"
TEXT: Final = {"type": "text", "text": "alpha bravo"}
EXPLANATION: Final = "This request was declined under the usage policy."
PENDING_SEARCH: Final = {
"type": "server_tool_use",
"id": "srvtoolu_pending",
"name": "web_search",
"input": {"query": "next city"},
}
TERMINAL_EVENTS: Final = ("response.completed", "response.incomplete", "response.failed")
def _sse(event: Mapping[str, JsonValue]) -> bytes:
@ -21,36 +30,64 @@ def _message(identity: str, stop_reason: str | None) -> dict[str, JsonValue]:
"type": "message",
"role": "assistant",
"model": MODEL,
"content": [{"type": "text", "text": "alpha bravo"}],
"content": [TEXT],
"stop_reason": stop_reason,
"stop_sequence": None,
"usage": {"input_tokens": 199878, "output_tokens": 12799},
}
def _stream(identity: str, stop_reason: str) -> tuple[bytes, ...]:
def _block_events(index: int, block: Mapping[str, JsonValue]) -> tuple[bytes, ...]:
if block["type"] == "text":
return (
_sse({"type": "content_block_start", "index": index, "content_block": {"type": "text", "text": ""}}),
_sse(
{
"type": "content_block_delta",
"index": index,
"delta": {"type": "text_delta", "text": block["text"]},
}
),
_sse({"type": "content_block_stop", "index": index}),
)
return (
_sse({"type": "message_start", "message": {**_message(identity, None), "content": []}}),
_sse({"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}),
_sse({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "alpha bravo"}}),
_sse({"type": "content_block_stop", "index": 0}),
_sse({"type": "content_block_start", "index": index, "content_block": {**block, "input": {}}}),
_sse(
{
"type": "message_delta",
"delta": {"stop_reason": stop_reason, "stop_sequence": None},
"usage": {"output_tokens": 12799},
"type": "content_block_delta",
"index": index,
"delta": {"type": "input_json_delta", "partial_json": json.dumps(block["input"])},
}
),
_sse({"type": "message_stop"}),
_sse({"type": "content_block_stop", "index": index}),
)
def _anthropic(identity: str, stop_reason: str):
def _turn(
identity: str,
stop_reason: str,
content: tuple[Mapping[str, JsonValue], ...] = (TEXT,),
stop_details: Mapping[str, JsonValue] | None = None,
):
stop: Final = {"stop_reason": stop_reason, "stop_sequence": None}
details: Final = {"stop_details": stop_details} if stop_details is not None else {}
message: Final = {**_message(identity, stop_reason), "content": list(content), **details}
stream: Final = (
_sse({"type": "message_start", "message": {**_message(identity, None), "content": []}}),
*(
event
for index, block in enumerate(content)
for event in _block_events(index, block) # comprehension-ok: block -> events flatten
),
_sse({"type": "message_delta", "delta": {**stop, **details}, "usage": {"output_tokens": 12799}}),
_sse({"type": "message_stop"}),
)
def respond(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/v1/messages", request.target
if json.loads(request.body).get("stream"):
return Reply(content_type="text/event-stream", chunks=_stream(identity, stop_reason))
return Reply(body=json.dumps(_message(identity, stop_reason)).encode())
return Reply(content_type="text/event-stream", chunks=stream)
return Reply(body=json.dumps(message).encode())
return respond
@ -67,45 +104,70 @@ def _events(gateway: Gateway, path: str, body: Mapping[str, JsonValue]) -> tuple
)
def _chat_finish_reason(gateway: Gateway, model: str, *, stream: bool) -> JsonValue:
body: Final = {"model": model, "max_tokens": 64000, "messages": [{"role": "user", "content": "copy the list"}]}
def _chat_choices(gateway: Gateway, body: Mapping[str, JsonValue], *, stream: bool) -> tuple[dict[str, JsonValue], ...]:
if stream:
reasons: Final = tuple(
choice["finish_reason"]
return tuple(
choice
for chunk in _events(gateway, "/v1/chat/completions", body)
for choice in chunk["choices"] # comprehension-ok: chunk -> choices flatten
if choice.get("finish_reason")
)
assert len(reasons) == 1, reasons
return reasons[0]
response: Final = gateway.request("POST", "/v1/chat/completions", body)
assert response.status_code == 200, response.text
return response.json()["choices"][0]["finish_reason"]
return tuple(response.json()["choices"])
def _responses_status(gateway: Gateway, model: str, *, stream: bool) -> JsonValue:
body: Final = {"model": model, "max_output_tokens": 64000, "input": "copy the list"}
def _chat_body(model: str) -> dict[str, JsonValue]:
return {"model": model, "max_tokens": 64000, "messages": [{"role": "user", "content": "copy the list"}]}
def _chat_finish_reason(gateway: Gateway, model: str, *, stream: bool) -> JsonValue:
reasons: Final = tuple(
choice["finish_reason"]
for choice in _chat_choices(gateway, _chat_body(model), stream=stream)
if choice.get("finish_reason")
)
assert len(reasons) == 1, reasons
return reasons[0]
def _chat_messages(gateway: Gateway, model: str, *, stream: bool) -> tuple[dict[str, JsonValue], ...]:
return tuple(
choice.get("delta") or choice.get("message") or {}
for choice in _chat_choices(gateway, _chat_body(model), stream=stream)
)
def _responses_body(model: str) -> dict[str, JsonValue]:
return {"model": model, "max_output_tokens": 64000, "input": "copy the list", "tools": [{"type": "web_search"}]}
def _responses_result(gateway: Gateway, model: str, *, stream: bool) -> dict[str, JsonValue]:
if stream:
terminal: Final = tuple(
event["response"]
for event in _events(gateway, "/v1/responses", body)
if event["type"] in ("response.completed", "response.incomplete", "response.failed")
for event in _events(gateway, "/v1/responses", _responses_body(model))
if event["type"] in TERMINAL_EVENTS
)
assert len(terminal) == 1, terminal
return terminal[0]["status"]
response: Final = gateway.request("POST", "/v1/responses", body)
return terminal[0]
response: Final = gateway.request("POST", "/v1/responses", _responses_body(model))
assert response.status_code == 200, response.text
return response.json()["status"]
return response.json()
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
@pytest.mark.parametrize(
("stop_reason", "finish_reason"),
[("model_context_window_exceeded", "length"), ("max_tokens", "length"), ("end_turn", "stop")],
[
("model_context_window_exceeded", "length"),
("max_tokens", "length"),
("pause_turn", "length"),
("end_turn", "stop"),
],
)
def test_chat_maps_anthropic_stop_reason(gateway: Gateway, stop_reason: str, finish_reason: str, stream: bool) -> None:
identity: Final = uuid.uuid4().hex
with wire_server(_anthropic(identity, stop_reason)) as wire, gateway.scenario() as scenario:
with wire_server(_turn(identity, stop_reason)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{MODEL}", api_base=wire.url, api_key="scripted-anthropic-key")
assert _chat_finish_reason(gateway, model, stream=stream) == finish_reason
@ -113,10 +175,81 @@ def test_chat_maps_anthropic_stop_reason(gateway: Gateway, stop_reason: str, fin
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
@pytest.mark.parametrize(
("stop_reason", "status"),
[("model_context_window_exceeded", "incomplete"), ("max_tokens", "incomplete"), ("end_turn", "completed")],
[
("model_context_window_exceeded", "incomplete"),
("max_tokens", "incomplete"),
("pause_turn", "completed"),
("end_turn", "completed"),
],
)
def test_responses_maps_anthropic_stop_reason(gateway: Gateway, stop_reason: str, status: str, stream: bool) -> None:
identity: Final = uuid.uuid4().hex
with wire_server(_anthropic(identity, stop_reason)) as wire, gateway.scenario() as scenario:
with wire_server(_turn(identity, stop_reason)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{MODEL}", api_base=wire.url, api_key="scripted-anthropic-key")
assert _responses_status(gateway, model, stream=stream) == status
assert _responses_result(gateway, model, stream=stream)["status"] == status
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
def test_chat_keeps_pause_turn_so_the_client_knows_to_resend_the_turn(gateway: Gateway, stream: bool) -> None:
identity: Final = uuid.uuid4().hex
paused: Final = _turn(identity, "pause_turn", (PENDING_SEARCH,))
with wire_server(paused) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{MODEL}", api_base=wire.url, api_key="scripted-anthropic-key")
stop_reasons: Final = tuple(
(message.get("provider_specific_fields") or {}).get("stop_reason")
for message in _chat_messages(gateway, model, stream=stream)
)
assert "pause_turn" in stop_reasons, stop_reasons
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
def test_responses_list_a_paused_turn_unrun_search_as_an_in_progress_web_search(gateway: Gateway, stream: bool) -> None:
identity: Final = uuid.uuid4().hex
paused: Final = _turn(identity, "pause_turn", (PENDING_SEARCH,))
with wire_server(paused) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{MODEL}", api_base=wire.url, api_key="scripted-anthropic-key")
result: Final = _responses_result(gateway, model, stream=stream)
assert (result["status"], result.get("incomplete_details")) == ("completed", None)
assert [(item["type"], item["status"], item["action"]["query"]) for item in result["output"]] == [
("web_search_call", "in_progress", PENDING_SEARCH["input"]["query"])
]
_REFUSAL: Final = {"type": "refusal", "category": "cyber", "explanation": EXPLANATION}
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
def test_chat_returns_the_refusal_explanation_as_the_message_refusal(gateway: Gateway, stream: bool) -> None:
identity: Final = uuid.uuid4().hex
with wire_server(_turn(identity, "refusal", (), _REFUSAL)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{MODEL}", api_base=wire.url, api_key="scripted-anthropic-key")
refusal: Final = "".join(
message["refusal"] for message in _chat_messages(gateway, model, stream=stream) if message.get("refusal")
)
assert refusal == EXPLANATION
@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"])
@pytest.mark.parametrize(
("content", "expected_parts"),
[
((), [("refusal", EXPLANATION)]),
((TEXT,), [("output_text", TEXT["text"]), ("refusal", EXPLANATION)]),
],
ids=["refusal_only", "answer_then_refusal"],
)
def test_responses_return_the_refusal_explanation_as_a_refusal_part(
gateway: Gateway,
content: tuple[Mapping[str, JsonValue], ...],
expected_parts: list[tuple[str, str]],
stream: bool,
) -> None:
identity: Final = uuid.uuid4().hex
with wire_server(_turn(identity, "refusal", content, _REFUSAL)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"anthropic/{MODEL}", api_base=wire.url, api_key="scripted-anthropic-key")
result: Final = _responses_result(gateway, model, stream=stream)
assert (result["status"], result["incomplete_details"]) == ("incomplete", {"reason": "content_filter"})
assert [
[(part["type"], part.get("text") or part.get("refusal")) for part in item["content"]]
for item in result["output"]
] == [expected_parts]

View file

@ -317,7 +317,6 @@ def test_credentialless_proxy_forwards_the_caller_anthropic_credential_and_never
def test_anthropic_sdk_metadata_user_id_reaches_upstream(api_key_proxy: Gateway, wire: Wire) -> None:
pytest.skip("BUG: /anthropic/v1/messages drops the native metadata.user_id field before forwarding upstream")
with api_key_proxy.scenario() as scenario:
key: Final = scenario.key()
sent: Final[list[httpx.Request]] = []

View file

@ -0,0 +1,123 @@
import json
import uuid
from collections.abc import Mapping
from dataclasses import dataclass
from typing import Final, Literal
import httpx
import pytest
from integration._support.client import Gateway, eventually, object_value
from integration._support.database import read_rows
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
_API_KEY: Final = "synthetic-decisions-key"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_INPUT: Final = "Listen to the attached clip."
_AUDIO: Final = ("data:audio/wav;base64,UklGRiQAAABXQVZF",)
_VIDEOS: Final = ("data:video/mp4;base64,AAAAIGZ0eXBpc29t",)
_QUESTION: Final[dict[str, JsonValue]] = {
"type": "predicate",
"name": "banana",
"instructions": "Does the clip name a banana?",
}
_SPEND_QUERY: Final = 'SELECT status, model_group FROM "LiteLLM_SpendLogs" WHERE request_id = %s'
Route = Literal["/v1/decisions", "/v1/systemone"]
Media = Literal["audio", "videos"]
@dataclass(frozen=True, slots=True)
class _Upstream:
model: str
def reply(self, request: Request) -> Reply:
questions: Final = object_value(_JSON_OBJECT.validate_json(request.body)["questions"])
plain: Final[dict[str, JsonValue]] = {
"model": self.model,
"answers": {name: {"type": "noul", "noul": 0.9} for name in questions},
"usage": {"input_tokens": 41, "output_tokens": 0},
}
return Reply(body=json.dumps(plain).encode())
def _unlisted_openrouter_model() -> _Upstream:
return _Upstream(f"openrouter/example/omni-{uuid.uuid4().hex[:12]}")
def _decide(gateway: Gateway, model: str, route: Route, media: Mapping[str, JsonValue]) -> httpx.Response:
body: Final[dict[str, JsonValue]] = (
{"model": model, "input": _INPUT, "questions": [_QUESTION], "num_retries": 0, **media}
if route == "/v1/decisions"
else {
"model": model,
"state": _INPUT,
"questions": {"banana": {"type": "noul", "instructions": _QUESTION["instructions"]}},
"num_retries": 0,
**media,
}
)
return gateway.request("POST", route, body)
def _sent_media(request: Request) -> dict[str, JsonValue]:
body: Final = _JSON_OBJECT.validate_json(request.body)
return {kind: body[kind] for kind in ("audio", "videos") if kind in body}
def _assert_refused(response: httpx.Response, kinds: tuple[Media, ...]) -> None:
assert response.status_code == 400, response.text
assert f"does not support parameters: {list(kinds)}" in response.text, response.text
_BOTH: Final[dict[str, JsonValue]] = {"audio": list(_AUDIO), "videos": list(_VIDEOS)}
@pytest.mark.parametrize("route", ("/v1/decisions", "/v1/systemone"))
def test_a_deployment_flagged_for_audio_and_video_sends_both_clips_for_a_model_the_cost_map_does_not_list(
gateway: Gateway, route: Route
) -> None:
upstream: Final = _unlisted_openrouter_model()
with wire_server(upstream.reply) as wire, gateway.scenario() as scenario:
flags: Final[dict[str, JsonValue]] = {"supports_audio_input": True, "supports_video_input": True}
flagged: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info=flags)
flagged_dropping: Final = scenario.model(
model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info=flags, drop_params=True
)
audio_only: Final = scenario.model(
model=upstream.model, api_base=wire.url, api_key=_API_KEY, model_info={"supports_audio_input": True}
)
unflagged: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY)
_assert_refused(_decide(gateway, unflagged, route, _BOTH), ("audio", "videos"))
_assert_refused(_decide(gateway, audio_only, route, _BOTH), ("videos",))
assert wire.drain() == ()
answered: Final = _decide(gateway, flagged, route, _BOTH)
assert answered.status_code == 200, answered.text
(call,) = wire.drain()
assert _sent_media(call) == _BOTH
rows: Final = eventually(
lambda: read_rows(_SPEND_QUERY, (answered.headers["x-litellm-call-id"],)),
lambda found: len(found) == 1,
seconds=70,
)
assert (rows[0]["status"], rows[0]["model_group"]) == ("success", flagged)
kept: Final = _decide(gateway, flagged_dropping, route, _BOTH)
assert kept.status_code == 200, kept.text
(kept_call,) = wire.drain()
assert _sent_media(kept_call) == _BOTH
@pytest.mark.parametrize(
"value",
({"supports_audio_input": True, "supports_video_input": True}, 5, [{"supports_audio_input": True}], "x" * 5000),
ids=("flags", "int", "list", "5kb"),
)
def test_a_caller_sent_model_info_never_turns_media_on(gateway: Gateway, value: JsonValue) -> None:
upstream: Final = _unlisted_openrouter_model()
with wire_server(upstream.reply) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=upstream.model, api_base=wire.url, api_key=_API_KEY)
_assert_refused(_decide(gateway, model, "/v1/decisions", {**_BOTH, "model_info": value}), ("audio", "videos"))
assert wire.drain() == ()

View file

@ -71,6 +71,13 @@ class FaultingTurnCheck(CustomLogger):
instance = FaultingTurnCheck()
"""
HEAVY_TURN_TOKENS: Final = 600
HEAVY_TURN_USAGE: Final[dict[str, JsonValue]] = {
"input_tokens": 400,
"output_tokens": 200,
"total_tokens": HEAVY_TURN_TOKENS,
}
SOCKET_TPM_LIMIT: Final = 1000
FrameReader = Callable[[], int]
@ -124,7 +131,16 @@ def _admitting_proxy(
(directory / f"{module}.py").write_text(source)
config: Final = _config(directory, router_settings, tuple(f"{module}.instance" for module in callback_sources))
with gateway_from_environment() as base:
with owned_proxy(base, directory, {"SSL_CERT_FILE": str(cert[0])}, config=config) as proxy:
with owned_proxy(
base,
directory,
{
"SSL_CERT_FILE": str(cert[0]),
"REDIS_HOST": os.environ["REDIS_HOST"],
"REDIS_PORT": os.environ["REDIS_PORT"],
},
config=config,
) as proxy:
yield proxy
@ -659,6 +675,159 @@ def test_usage_based_routing_refuses_a_turn_only_once_earlier_turns_used_up_the_
assert deployment.forwarded_turns() == 2
def _heavy_turn(response_id: str) -> tuple[dict[str, JsonValue], ...]:
*started, done = events(response_id, "heavy")
return (*started, {**done, "response": {**object_value(done["response"]), "usage": HEAVY_TURN_USAGE}})
def _counted_tokens(counter: str) -> int:
with _redis() as redis:
return int(redis.get(counter) or 0)
def _chat(gateway: Gateway, key: str, model: str, headers: Mapping[str, str] | None = None) -> httpx.Response:
return gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": f"after the socket {uuid.uuid4().hex}"}]},
key=key,
headers=headers,
)
def _two_heavy_turns_then_chat(
gateway: Gateway,
cert: tuple[Path, Path],
counter: Callable[[str], str],
headers: Mapping[str, str],
**limits: JsonValue,
) -> httpx.Response:
with responses_peer(cert, _heavy_turn) as peer, gateway.scenario() as scenario:
deployment: Final = _native(scenario, peer)
chat: Final = scenario.model()
key: Final = scenario.key(models=[deployment.name, chat], **limits)
_fresh_minute()
async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]:
async with _socket(gateway, key, deployment.name, headers) as connection:
return (await _say(connection, deployment.name), await _say(connection, deployment.name))
turns: Final = asyncio.run(session())
assert [completed(received)["status"] for received in turns] == ["completed"] * 2
eventually(
partial(_counted_tokens, counter(key)),
lambda tokens: tokens >= 2 * HEAVY_TURN_TOKENS,
seconds=15,
)
return _chat(gateway, key, chat, headers)
def test_websocket_turn_tokens_count_toward_the_key_tpm_so_its_next_http_call_is_refused(
limited: Gateway, cert: tuple[Path, Path]
) -> None:
refused: Final = _two_heavy_turns_then_chat(
limited, cert, lambda key: f"{{api_key:{_key_hash(key)}}}:tokens", {}, tpm_limit=SOCKET_TPM_LIMIT
)
assert refused.status_code == 429, refused.text
assert "token" in refused.text.lower(), refused.text
def test_websocket_turn_tokens_count_toward_the_tag_tpm_so_its_next_http_call_is_refused(
limited: Gateway, cert: tuple[Path, Path]
) -> None:
tag: Final = f"ws-tag-tpm-{uuid.uuid4().hex}"
limited.post("/tag/new", {"name": tag, "tpm_limit": SOCKET_TPM_LIMIT})
try:
refused: Final = _two_heavy_turns_then_chat(
limited, cert, lambda _: f"{{tag:{tag}}}:tokens", {"x-litellm-tags": tag}
)
finally:
limited.post("/tag/delete", {"name": tag})
assert refused.status_code == 429, refused.text
assert "tag" in refused.text.lower(), refused.text
def _key_tokens(key: str) -> int:
return _counted_tokens(f"{{api_key:{_key_hash(key)}}}:tokens")
def test_a_turn_is_refused_on_the_open_socket_once_earlier_turns_used_up_the_key_tpm(
limited: Gateway, cert: tuple[Path, Path]
) -> None:
with responses_peer(cert, _heavy_turn) as peer, limited.scenario() as scenario:
deployment: Final = _native(scenario, peer)
key: Final = scenario.key(models=[deployment.name], tpm_limit=SOCKET_TPM_LIMIT)
async def session() -> tuple[tuple[tuple[dict[str, JsonValue], ...], ...], int | None]:
async with _socket(limited, key, deployment.name) as connection:
answered: Final = (await _say(connection, deployment.name), await _say(connection, deployment.name))
await _until(partial(_key_tokens, key), lambda tokens: tokens >= 2 * HEAVY_TURN_TOKENS)
refused: Final = (await _say(connection, deployment.name), await _say(connection, deployment.name))
return (*answered, *refused), connection.close_code
(first, second, third, fourth), close_code = asyncio.run(session())
assert [completed(received)["status"] for received in (first, second)] == ["completed"] * 2
_assert_refused(third)
_assert_refused(fourth)
assert close_code is None
assert deployment.forwarded_turns() == 2
def test_managed_socket_turns_count_toward_the_sockets_tag_tpm_like_the_key_tpm(limited: Gateway) -> None:
tag: Final = f"ws-managed-tag-tpm-{uuid.uuid4().hex}"
limited.post("/tag/new", {"name": tag, "tpm_limit": 1_000_000})
try:
with limited.scenario() as scenario:
deployment: Final = _managed(scenario)
key: Final = scenario.key(models=[deployment.name], tpm_limit=1_000_000)
async def session() -> tuple[tuple[dict[str, JsonValue], ...], ...]:
async with _socket(limited, key, deployment.name, {"x-litellm-tags": tag}) as connection:
return (await _say(connection, deployment.name), await _say(connection, deployment.name))
turns: Final = asyncio.run(session())
turn_tokens: Final = sum(
int(str(object_value(completed(received)["usage"])["total_tokens"])) for received in turns
)
assert turn_tokens > 0, turns
eventually(
lambda: (_key_tokens(key), _counted_tokens(f"{{tag:{tag}}}:tokens")),
lambda counters: counters[0] >= turn_tokens and counters[0] == counters[1],
seconds=20,
)
finally:
limited.post("/tag/delete", {"name": tag})
def test_a_socket_opened_after_turns_used_up_the_key_tpm_gets_the_turn_refusal_frame_and_closes_1008(
limited: Gateway, cert: tuple[Path, Path]
) -> None:
with responses_peer(cert, _heavy_turn) as peer, limited.scenario() as scenario:
deployment: Final = _native(scenario, peer)
key: Final = scenario.key(models=[deployment.name], tpm_limit=SOCKET_TPM_LIMIT)
async def spend_then_reopen() -> tuple[tuple[dict[str, JsonValue], ...], int | None]:
async with _socket(limited, key, deployment.name) as spending:
completed(await _say(spending, deployment.name))
completed(await _say(spending, deployment.name))
await _until(partial(_key_tokens, key), lambda tokens: tokens >= 2 * HEAVY_TURN_TOKENS)
async with _socket(limited, key, deployment.name) as refused:
received: Final = await _say(refused, deployment.name)
await asyncio.wait_for(refused.wait_closed(), timeout=20)
return received, refused.close_code
received, close_code = asyncio.run(spend_then_reopen())
_assert_refused(received)
assert close_code == 1008, close_code
assert deployment.forwarded_turns() == 2
def _assert_http_chat_counted_once(gateway: Gateway) -> None:
with gateway.scenario() as scenario:
identity: Final = f"ws-admission-{uuid.uuid4().hex}"

Some files were not shown because too many files have changed in this diff Show more