scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-14 09:03:16 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 214

chore(merge): merge main e60e642290 into litellm_openai_sdk3_docker
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
Details
LiteLLM Rust / rust-test (push) Has been cancelled
Details
LiteLLM Rust / rust-wheel (push) Has been cancelled
Details
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Details
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Details
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Details
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Details
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
Details
LiteLLM Rust DB / db-queries (push) Has been cancelled
Details

Browse source 
Resolve the conflicts from the #49986 typing refactor in exceptions.py,
azure.py, responses/utils.py, and test_responses_utils.py by keeping both
sides.

Dump Responses output items and tools by their wire names, so SDK 3's
async_ field goes out as async, and replay streamed tool search items with
only the fields upstream sent. Update the Arize span tests for the SDK 3
tool fields.
This commit is contained in:
mateo-berri 2026-11-06 17:51:58 -08:00
commit d3c0ecd063
411 changed files with 13417 additions and 2208 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
.github/workflows/test-unit.yml vendored
View file

@ -1035,9 +1035,9 @@ jobs:
run: |
diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
else
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
@ -1294,7 +1294,7 @@ jobs:
- name: Install dependencies
if: steps.changes.outputs.decision != 'skip'
run: |
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard
- name: Cache Prisma binaries
if: steps.changes.outputs.decision != 'skip'
@ -1497,9 +1497,9 @@ jobs:
RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }}
run: |
if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
else
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy
.github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy
uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
fi

4
enterprise/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.81"
version = "0.1.83"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.81"
version = "0.1.83"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

4
litellm-proxy-extras/pyproject.toml
View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.112"
version = "0.4.114"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.112"
version = "0.4.114"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

28
litellm-rust/crates/python-bridge/src/router/host.rs
View file

@ -20,10 +20,10 @@ use std::{
use async_trait::async_trait;
use litellm_router_core::{
Attempt, AttemptError, Blocked, BuildAsk, BuildPath, Built, Deployment, Error, Host, Output,
Payload, PreRoute, PreRouted, Provided, Randomness, RouterCache, RouterEvent, RouterRequest,
RouterResponse, RoutingContext, SecretRef, SelectStage, ThreadRandomness,
build::BuiltDeployment, host::ModelInfoLookup, model_info::MergedModelInfo,
Attempt, AttemptError, Blocked, BuildAsk, BuildPath, Built, CallbackFiltered, Deployment,
Error, Host, Output, Payload, PreRoute, PreRouted, Provided, Randomness, RouterCache,
RouterEvent, RouterRequest, RouterResponse, RoutingContext, SecretRef, SelectStage,
ThreadRandomness, build::BuiltDeployment, host::ModelInfoLookup, model_info::MergedModelInfo,
precall::PreCallFacts,
};
use pyo3::{PyResult, Python, types::PyAnyMethods};
@ -298,8 +298,9 @@ pub(crate) enum HostCallResponse {
PreCall(PreCallFacts),
/// The candidate models a `routing_plugins` select step reported.
Models(Option<Vec<String>>),
/// The ids a `callback_filters` select step left, or the error a filter raised.
Filtered(Result<Vec<String>, AttemptError>),
/// The ids a `callback_filters` select step left with the one a filter prefers, or the error
/// a filter raised.
Filtered(Result<(Vec<String>, Option<String>), AttemptError>),
/// What a pick's select or pre-route step, or a fallback check, raised on the host instead of
/// answering: the walk takes it as the try's failure.
Raised(AttemptError),
@ -352,8 +353,10 @@ impl HostCallResponse {
got: self.kind(),
});
}
if let (Self::Select(ids) | Self::Filtered(Ok(ids)), HostCall::Select { candidates, .. }) =
(self, call)
if let (
Self::Select(ids) | Self::Filtered(Ok((ids, _))),
HostCall::Select { candidates, .. },
) = (self, call)
&& let Some(id) = ids.iter().find(|id| !candidates.contains(id))
{
return Err(DriveError::UnknownCandidate { id: id.clone() });
@ -890,7 +893,7 @@ impl Host for StepHost {
&self,
ctx: &RoutingContext<'_>,
candidates: Vec<Arc<Deployment>>,
) -> Result<Vec<Arc<Deployment>>, Error> {
) -> Result<CallbackFiltered, Error> {
let (scope, group) = ctx.scope.wire();
let joined = |own: &[String], walked: &[String]| [own, walked].concat();
let call = HostCall::Select {
@ -909,8 +912,11 @@ impl Host for StepHost {
let HostCallResponse::Filtered(result) = call_host(call).await else {
unreachable!("{CHECKED}");
};
let ids = result.map_err(Error::Attempt)?;
Ok(answered_by_id(&candidates, &ids))
let (ids, preferred) = result.map_err(Error::Attempt)?;
Ok(CallbackFiltered {
kept: answered_by_id(&candidates, &ids),
preferred,
})
}
async fn allow_fallback(

8
litellm-rust/crates/python-bridge/src/router/mod.rs
View file

@ -1181,7 +1181,8 @@ fn host_call_response(
stage: Stage::CallbackFilters { .. },
..
} => {
// The remaining ids, or a failure mapping as an attempt reports one.
// The remaining ids, alone or paired with the id a filter prefers, or a failure
// mapping as an attempt reports one.
let outcome = outcome.ok_or_else(missing)?;
if let Ok(failure) = outcome.cast::<PyDict>() {
return match attempt_outcome(failure)? {
@ -1191,7 +1192,10 @@ fn host_call_response(
)),
};
}
Ok(HostCallResponse::Filtered(Ok(outcome.extract()?)))
if let Ok(answer) = outcome.cast::<PyTuple>() {
return Ok(HostCallResponse::Filtered(Ok(answer.extract()?)));
}
Ok(HostCallResponse::Filtered(Ok((outcome.extract()?, None))))
}
HostCall::Select {
stage: Stage::Strategy { records, .. },

19
litellm-rust/crates/router-core/src/host.rs
View file

@ -405,6 +405,12 @@ pub enum SelectStage {
OverrideCheck(RoutingStrategy),
}
#[derive(Debug)]
pub struct CallbackFiltered {
pub kept: Vec<Arc<Deployment>>,
pub preferred: Option<String>,
}
/// Routing events, delivered through [`Host::observe`]. They serialize as JSON objects tagged by
/// `event` (the variant name), durations in seconds.
#[derive(Clone, Debug, PartialEq, serde::Serialize)]
@ -714,14 +720,19 @@ pub trait Host: Send + Sync {
}
/// `async_callback_filter_deployments`: the host's routing callbacks narrow `candidates` in
/// turn. A filter that raises fails the try with [`Error::Attempt`], which the call retries and
/// falls back from like any attempt failure. A host without filters keeps every candidate.
/// turn, and may name one deployment they prefer, which the pick takes only when the filters
/// after the callbacks keep it. A filter that raises fails the try with [`Error::Attempt`],
/// which the call retries and falls back from like any attempt failure. A host without
/// filters keeps every candidate and prefers none.
async fn callback_filters(
&self,
_ctx: &RoutingContext<'_>,
candidates: Vec<Arc<Deployment>>,
) -> Result<Vec<Arc<Deployment>>, Error> {
Ok(candidates)
) -> Result<CallbackFiltered, Error> {
Ok(CallbackFiltered {
kept: candidates,
preferred: None,
})
}
/// The values of `refs`, one per ref, in order: what the deployment build substitutes for
/// each `os.environ/` reference it found in entry `index` ([`crate::build`]). Asked once per

4
litellm-rust/crates/router-core/src/lib.rs
View file

@ -124,8 +124,8 @@ pub use crate::{
error::{AttemptError, ConfigError, Error, FailureKind, Payload, Unsupported},
health::{Clock, Health, Outcome, SystemClock},
host::{
Attempt, Blocked, BuildAsk, BuildPath, Built, Host, Output, Provided, RetryCause,
RouterEvent, RouterRequest, RoutingContext, SecretAt, SecretRef, SelectStage,
Attempt, Blocked, BuildAsk, BuildPath, Built, CallbackFiltered, Host, Output, Provided,
RetryCause, RouterEvent, RouterRequest, RoutingContext, SecretAt, SecretRef, SelectStage,
},
outcome::{OutcomeCallbacks, OutcomeWrite},
prerouting::{PreRoute, PreRouted},

19
litellm-rust/crates/router-core/src/select.rs
View file

@ -456,7 +456,7 @@ impl Core {
let request_order = request.target_order.filter(|_| !inputs.fallback_hop);
// `async_callback_filter_deployments` runs after the cooldown and blocked filters on every
// async candidate list but a deployment named by id and a listed pass-through pick.
if settings.callback_filters
let preferred = if settings.callback_filters
&& !inputs.count_only
&& !resolution.kind.is_single()
&& request.pass_through != Some(PassThrough::Listed)
@ -472,8 +472,12 @@ impl Core {
ctx.excluded_ids = &inputs.excluded_ids;
ctx.skipped_ids = &inputs.skipped_ids;
ctx.target_order = inputs.target_order.or(request_order);
healthy = self.host.callback_filters(&ctx, healthy).await?;
}
let filtered = self.host.callback_filters(&ctx, healthy).await?;
healthy = filtered.kept;
filtered.preferred
} else {
None
};
if inputs.count_only {
return Ok(Candidates {
name: name.to_owned(),
@ -648,8 +652,13 @@ impl Core {
healthy.retain(|d| !d.cannot_run_batches());
}
if let Some(pinned) = request.pinned_deployment_id.as_deref()
&& healthy.iter().any(|d| d.id == pinned)
if let Some(pinned) = [
request.pinned_deployment_id.as_deref(),
preferred.as_deref(),
]
.into_iter()
.flatten()
.find(|id| healthy.iter().any(|d| d.id == *id))
{
healthy.retain(|d| d.id == pinned);
}

37
litellm/a2a_protocol/card_resolver.py
View file

@ -4,13 +4,14 @@ Custom A2A Card Resolver for LiteLLM.
Extends the A2A SDK's card resolver to support multiple well-known paths.
"""
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Protocol, runtime_checkable
from litellm._logging import verbose_logger
from litellm.a2a_protocol.exceptions import A2AAgentCardDiscoveryError
from litellm.constants import LOCALHOST_URL_PATTERNS
from litellm.litellm_core_utils.core_helpers import has_attributes
if TYPE_CHECKING:
from a2a.types import AgentCard
@ -130,15 +131,34 @@ def normalize_agent_card_interfaces(agent_card: "AgentCard", protocol_version: s
return normalized
class _UrlCarrier(Protocol):
url: str
class _InterfacesCarrier(Protocol):
@property
def supported_interfaces(self) -> Sequence[_UrlCarrier]: ...
def _url_of(value: object) -> str | None:
return value.url if has_attributes(value, _UrlCarrier, "url") else None
def _supported_interfaces(agent_card: object) -> Sequence[_UrlCarrier] | None:
if has_attributes(agent_card, _InterfacesCarrier, "supported_interfaces"):
return agent_card.supported_interfaces
return None
def get_agent_card_url(agent_card: "AgentCard") -> str | None:
"""Return the agent endpoint URL from the resolved SDK card."""
url: Final = getattr(agent_card, "url", None)
url: Final = _url_of(agent_card)
if url:
return url
interfaces: Final = getattr(agent_card, "supported_interfaces", None)
interfaces: Final = _supported_interfaces(agent_card)
if interfaces:
return getattr(interfaces[0], "url", None)
return _url_of(interfaces[0])
return None
@ -168,16 +188,16 @@ def fix_agent_card_url(agent_card: "AgentCard", base_url: str) -> "AgentCard":
Returns:
The agent card with the URL fixed if necessary
"""
card_url: Final = getattr(agent_card, "url", None)
card_url: Final = _url_of(agent_card)
if card_url and is_localhost_or_internal_url(card_url):
# Normalize base_url to ensure it ends with /
fixed_url: Final = base_url.rstrip("/") + "/"
agent_card.url = fixed_url
interfaces: Final = getattr(agent_card, "supported_interfaces", None)
interfaces: Final = _supported_interfaces(agent_card)
if interfaces:
interface_url: Final = getattr(interfaces[0], "url", None)
interface_url: Final = _url_of(interfaces[0])
if interface_url and is_localhost_or_internal_url(interface_url):
interfaces[0].url = base_url.rstrip("/") + "/"
@ -194,6 +214,9 @@ class LiteLLMA2ACardResolver(_A2ACardResolver):
- /agentCard/v1.0
"""
if TYPE_CHECKING:
base_url: str
async def get_agent_card(
self,
relative_card_path: str | None = None,

5
litellm/a2a_protocol/exception_mapping_utils.py
View file

@ -4,7 +4,7 @@ A2A Protocol Exception Mapping Utils.
Maps A2A SDK exceptions to LiteLLM A2A exception types.
"""
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Final
from litellm._logging import verbose_logger
from litellm.a2a_protocol.card_resolver import (
@ -21,6 +21,7 @@ from litellm.constants import CONNECTION_ERROR_PATTERNS
if TYPE_CHECKING:
from a2a.client import Client as A2AClientType
from a2a.types import AgentCard
_A2A_KIND_DIALECT_HINT: Final = (
@ -165,7 +166,7 @@ def map_a2a_exception(
async def handle_a2a_localhost_retry(
error: A2ALocalhostURLError,
agent_card: Any,
agent_card: "AgentCard | None",
a2a_client: "A2AClientType",
is_streaming: bool = False,
) -> "A2AClientType":

7
litellm/a2a_protocol/litellm_completion_bridge/handler.py
View file

@ -10,7 +10,7 @@ A2A Streaming Events (in order):
4. Status update (kind: "status-update") - Final status "completed" with final=true
"""
from collections.abc import AsyncIterator, Callable, Coroutine, Mapping
from collections.abc import AsyncIterator, Mapping
from types import MappingProxyType
from typing import Any, Final
@ -126,10 +126,7 @@ class A2ACompletionBridgeHandler:
@staticmethod
async def _acompletion(completion_params: Mapping[str, object]) -> ModelResponse | CustomStreamWrapper:
acompletion_fn: Final[Callable[..., Coroutine[object, object, ModelResponse | CustomStreamWrapper]]] = vars(
litellm
)["acompletion"]
return await acompletion_fn(**completion_params)
return await litellm.acompletion(**completion_params)
@staticmethod
async def handle_non_streaming(

15
litellm/a2a_protocol/main.py
View file

@ -14,7 +14,7 @@ import datetime
import uuid
from collections.abc import AsyncIterator, Coroutine, Mapping
from types import MappingProxyType, ModuleType
from typing import TYPE_CHECKING, Final, Optional, cast
from typing import TYPE_CHECKING, Final, Optional, Protocol, cast
from typing_extensions import TypedDict, Unpack
@ -24,6 +24,7 @@ from litellm.a2a_protocol.streaming_iterator import A2AStreamingIterator
from litellm.a2a_protocol.utils import A2ARequestUtils
from litellm.constants import DEFAULT_A2A_AGENT_TIMEOUT
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
@ -36,7 +37,6 @@ if TYPE_CHECKING:
from a2a.client import Client as A2AClientType
from a2a.client import ClientCallContext as A2ACallContextType
from a2a.compat.v0_3.types import (
AgentCard,
Message,
SendMessageRequest,
SendMessageResponse,
@ -45,6 +45,7 @@ if TYPE_CHECKING:
SendStreamingMessageSuccessResponse,
Task,
)
from a2a.types import AgentCard
from a2a.types.a2a_pb2 import SendMessageRequest as CoreSendMessageRequest
from a2a.types.a2a_pb2 import StreamResponse as CoreStreamResponse
@ -95,6 +96,12 @@ class _SendMessageOptions(TypedDict, total=False, extra_items=object):
pass
class _StreamingRequestConverter(Protocol):
def to_core_send_message_request(
self, compat_req: "SendStreamingMessageRequest", /
) -> "CoreSendMessageRequest": ...
def _set_usage_on_logging_obj(
kwargs: Mapping[str, object],
prompt_tokens: int,
@ -375,7 +382,9 @@ async def _stream_messages(
"The 'a2a' package is required for A2A agent invocation. Install it with: pip install a2a-sdk"
)
pb_request: Final[CoreSendMessageRequest] = _a2a_conversions.to_core_send_message_request(request)
pb_request: Final[CoreSendMessageRequest] = require_attributes(
_a2a_conversions, _StreamingRequestConverter, "to_core_send_message_request"
).to_core_send_message_request(request)
async for event in a2a_client.send_message(pb_request, context=_get_a2a_call_context(a2a_client)):
compat_chunk = _to_compat_stream_response(event, request_id=request.id)
yield SendStreamingMessageResponse(root=compat_chunk)

25
litellm/caching/caching.py
View file

@ -13,10 +13,10 @@ import json
import logging
import time
import traceback
from collections.abc import Generator, Mapping
from collections.abc import Awaitable, Callable, Generator, Mapping
from contextlib import contextmanager
from enum import Enum
from typing import Any, Final, Literal, cast
from typing import Any, Final, Literal, Protocol, cast
from pydantic import BaseModel, ConfigDict, TypeAdapter
@ -25,6 +25,7 @@ from litellm._internal_context import current_service_target, service_target
from litellm._logging import verbose_logger
from litellm.constants import CACHED_STREAMING_CHUNK_DELAY
from litellm.integrations.otel.runtime import phase_span
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.model_param_helper import (
ModelParamHelper,
embeds_thought_signature_in_tool_call_id,
@ -52,6 +53,16 @@ _REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter(
)
class _SupportsAsyncPing(Protocol):
@property
def ping(self) -> Callable[[], Awaitable[object]] | None: ...
class _SupportsAsyncDeleteCacheKeys(Protocol):
@property
def delete_cache_keys(self) -> Callable[[object], Awaitable[object]] | None: ...
def _keyed_provider_params_in_extra_body(kwargs: Mapping[str, object]) -> str:
raw_extra_body: Final = kwargs.get("extra_body")
if not isinstance(raw_extra_body, dict):
@ -1032,14 +1043,16 @@ class Cache:
cache_key, cached_data, kwargs = self._add_cache_logic(result=result, **kwargs)
await self.cache.batch_cache_write(cache_key, cached_data, **kwargs)
async def ping(self):
cache_ping: Final = getattr(self.cache, "ping")
async def ping(self) -> object:
cache_ping: Final = require_attributes(self.cache, _SupportsAsyncPing, "ping").ping
if cache_ping:
return await cache_ping()
return None
async def delete_cache_keys(self, keys):
cache_delete_cache_keys: Final = getattr(self.cache, "delete_cache_keys")
async def delete_cache_keys(self, keys: object) -> object:
cache_delete_cache_keys: Final = require_attributes(
self.cache, _SupportsAsyncDeleteCacheKeys, "delete_cache_keys"
).delete_cache_keys
if cache_delete_cache_keys:
return await cache_delete_cache_keys(keys)
return None

39
litellm/caching/caching_handler.py
View file

@ -22,6 +22,7 @@ from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable,
from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar
from pydantic import ConfigDict, SkipValidation, TypeAdapter, ValidationError
from typing_extensions import TypedDict
import litellm
from litellm._internal_context import is_internal_call, post_response_phase
@ -90,6 +91,10 @@ class CachingHandlerResponse(LiteLLMBaseModel):
in_memory_cache_obj: Final = InMemoryCache()
class _CacheCallKwargs(TypedDict, total=False, extra_items=object):
pass
def _drop_logging_obj_from_kwargs(request_kwargs: dict[str, object]) -> dict[str, object]:
"""
The caching handler is stored on the Logging object
@ -455,7 +460,7 @@ class LLMCachingHandler:
if litellm.cache is not None and self._is_call_type_supported_by_cache(original_function=original_function):
args = args or ()
# Now that we confirmed caching will happen, prepare kwargs
new_kwargs: Final = kwargs.copy()
new_kwargs: Final = dict[str, object](kwargs)
new_kwargs.update(
convert_args_to_kwargs(
self.original_function,
@ -470,7 +475,7 @@ class LLMCachingHandler:
print_verbose("Checking Sync Cache")
logging_obj.reset_caching_details()
with response_cache_phase("get"):
cached_result = litellm.cache.get_cache(**new_kwargs)
cached_result = litellm.cache.get_cache(**_CacheCallKwargs(**new_kwargs))
if is_response_without_output(cached_result):
verbose_logger.debug("LiteLLM Cache: cached response has no output, treating it as a miss")
return CachingHandlerResponse(cached_result=None)
@ -480,7 +485,7 @@ class LLMCachingHandler:
pass
else:
call_type = original_function.__name__
cached_result = self._convert_cached_result_to_model_response(
converted_result: Final = self._convert_cached_result_to_model_response(
cached_result=cached_result,
call_type=call_type,
kwargs=kwargs,
@ -508,16 +513,16 @@ class LLMCachingHandler:
logging_obj=logging_obj,
model=f"{custom_llm_provider}/{model}",
kwargs=kwargs,
cached_result=cached_result,
cached_result=converted_result,
is_async=False,
custom_llm_provider=custom_llm_provider,
)
if not is_internal_call.get() and not _should_defer_streaming_cache_hit_callbacks(
cached_result=cached_result
cached_result=converted_result
):
logging_obj.handle_sync_cache_hit_success_callbacks(
result=cached_result,
result=converted_result,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
@ -527,8 +532,8 @@ class LLMCachingHandler:
or self.request_kwargs.get("cache_key")
or litellm.cache.get_cache_key(**self.request_kwargs)
)
_set_cached_hidden_param(cached_result, "cache_key", cache_key)
return CachingHandlerResponse(cached_result=cached_result)
_set_cached_hidden_param(converted_result, "cache_key", cache_key)
return CachingHandlerResponse(cached_result=converted_result)
return CachingHandlerResponse(cached_result=cached_result)
_sync_get_cache = sync_get_cache
@ -950,22 +955,14 @@ class LLMCachingHandler:
def _convert_cached_result_to_model_response(
self,
cached_result: Any,
cached_result: object,
call_type: str,
kwargs: dict[str, object],
logging_obj: LiteLLMLoggingObj,
model: str,
args: tuple[object, ...],
custom_llm_provider: str | None = None,
) -> (
ModelResponse
| TextCompletionResponse
| EmbeddingResponse
| RerankResponse
| TranscriptionResponse
| CustomStreamWrapper
| None
):
) -> object:
"""
Internal method to process the cached result
@ -1141,7 +1138,7 @@ class LLMCachingHandler:
self,
result: object,
original_function: Callable,
kwargs: dict[str, Any],
kwargs: Mapping[str, object],
args: tuple[object, ...] | None = None,
):
"""
@ -1169,7 +1166,7 @@ class LLMCachingHandler:
return
cache: Final = litellm.cache
new_kwargs: Final = kwargs.copy()
new_kwargs: Final = dict(kwargs)
new_kwargs.update(
convert_args_to_kwargs(
original_function,
@ -1206,7 +1203,7 @@ class LLMCachingHandler:
)
)
else:
create_cache_write_task(lambda: cache.async_add_cache(result, **new_kwargs))
create_cache_write_task(lambda: cache.async_add_cache(result, **_CacheCallKwargs(**new_kwargs)))
def sync_set_cache(
self,

27
litellm/caching/redis_cache.py
View file

@ -40,7 +40,7 @@ from litellm.constants import (
REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION,
REDIS_TIMEOUT_LOG_INTERVAL,
)
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs
from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs, has_attributes
from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
from litellm.redis.store_settings import RedisStoreSettings, breaker_settings
from litellm.types.caching import (
@ -97,6 +97,22 @@ class _AsyncRedisCommands(Protocol):
def eval(self, script: str, numkeys: int, *keys_and_args: str | bytes | float) -> Awaitable[object]: ...
class _AsyncLuaScript(Protocol):
def __call__(
self, keys: Sequence[str], args: Sequence[str | bytes | int | float], client: object
) -> Awaitable[object]: ...
class _SupportsRegisterScript(Protocol):
def register_script(self, script: str, /) -> _AsyncLuaScript: ...
class _SupportsScriptLoad(Protocol):
def script_load(self, script: str, /) -> object: ...
def evalsha(self, sha: object, numkeys: int, /, *keys_and_args: str | bytes | float) -> Awaitable[object]: ...
_GENERIC_CALLER_MODULES: Final = frozenset(
{
__name__,
@ -844,6 +860,9 @@ class RedisCache(BaseCache):
def _async_commands(self) -> _AsyncRedisCommands:
return self.init_async_client()
def _async_client_object(self) -> object:
return self.init_async_client()
def check_and_fix_namespace(self, key: str) -> str:
"""
Make sure each key starts with the given namespace
@ -1117,8 +1136,8 @@ class RedisCache(BaseCache):
Kept separate from async_register_script so each loop caches its own
executor; see that method for why the binding must be per loop.
"""
_redis_client: Final[Any] = self.init_async_client()
if hasattr(_redis_client, "register_script"):
_redis_client: Final = self._async_client_object()
if has_attributes(_redis_client, _SupportsRegisterScript, "register_script"):
registered_script: Final = _redis_client.register_script(script)
async def standalone_executor(
@ -1131,7 +1150,7 @@ class RedisCache(BaseCache):
return standalone_executor
if hasattr(_redis_client, "script_load"):
if has_attributes(_redis_client, _SupportsScriptLoad, "script_load"):
script_sha: Final = _redis_client.script_load(script)
async def cluster_executor(

5
litellm/caching/redis_semantic_cache.py
View file

@ -43,6 +43,7 @@ if TYPE_CHECKING:
_CACHE_RESPONSE_ADAPTER: Final[TypeAdapter[str | bytes | None]] = TypeAdapter(
str | bytes | None, config=ConfigDict(strict=True, hide_input_in_errors=True)
)
_DECODED_CACHE_VALUE: Final = TypeAdapter(object)
class RedisSemanticCache(BaseCache):
@ -407,10 +408,10 @@ class RedisSemanticCache(BaseCache):
# Convert string representation to Python object
try:
return json.loads(response_text)
return _DECODED_CACHE_VALUE.validate_python(json.loads(response_text))
except json.JSONDecodeError:
try:
return ast.literal_eval(response_text)
return _DECODED_CACHE_VALUE.validate_python(ast.literal_eval(response_text))
except (ValueError, SyntaxError) as e:
print_verbose(f"Error parsing cached response: {e}")
return None

13
litellm/completion_extras/litellm_responses_transformation/transformation.py
View file

@ -39,7 +39,7 @@ from litellm.litellm_core_utils.allowed_tools_tool_choice import (
parse_chat_allowed_tools_tool_choice,
responses_allowed_tools_tool_choice,
)
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, iter_items
from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
prompt_cache_breakpoint_with_readable_mode,
@ -349,12 +349,15 @@ _TOOL_SEARCH_REPLAY_ITEM_TYPES: Final = frozenset({"reasoning", *_TOOL_SEARCH_IT
_TOOL_SEARCH_ITEMS_KEY: Final = "tool_search_items"
_DECODED_JSON: Final = TypeAdapter(object)
def _json_field(value: object, key: str) -> object:
return cast(Mapping[str, object], value).get(key) if isinstance(value, dict) else None # cast-ok: untyped JSON
def _tool_search_replay_item(item: object) -> Mapping[str, object] | None:
raw: Final[object] = item.model_dump(exclude_unset=True) if isinstance(item, BaseModel) else item
raw: Final[object] = item.model_dump(exclude_unset=True, by_alias=True) if isinstance(item, BaseModel) else item
if not isinstance(raw, dict):
return None
fields: Final = cast(dict[str, object], raw) # cast-ok: provider JSON item, keys forwarded verbatim
@ -857,8 +860,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
# Handle message items with output_text content
if item_type == "message":
content_list: Final = item.get("content", [])
for content_item in content_list:
content_list: Final = _DECODED_JSON.validate_python(item.get("content", []))
for content_item in iter_items(content_list):
if isinstance(content_item, dict):
content_type = content_item.get("type")
if content_type == "output_text":
@ -2050,7 +2053,7 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
from pydantic import BaseModel
if isinstance(str_line, BaseModel):
return self.chunk_parser(str_line.model_dump())
return self.chunk_parser(str_line.model_dump(exclude_unset=True))
if not str_line or str_line.startswith("event:"):
# ignore.

33
litellm/cost_calculator.py
View file

@ -8,7 +8,7 @@ from datetime import datetime
from functools import lru_cache
from itertools import accumulate
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, cast
from httpx import Response
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
@ -24,7 +24,12 @@ from litellm.constants import (
REALTIME_METERED_USAGE_KEY,
)
from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, billed_usage_cost
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
attribute_or_none,
billed_usage_cost,
require_attributes,
)
from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
from litellm.litellm_core_utils.decisions_pricing import (
decisions_pricing_fields,
@ -1188,6 +1193,10 @@ def _model_contains_known_llm_provider(model: str) -> bool:
return _provider_prefix in LlmProvidersSet
class _SupportsGet(Protocol):
def get(self, key: str, /) -> object: ...
def _get_response_model(completion_response: object) -> str | None:
"""
Extract the model name from a completion response object.
@ -1219,7 +1228,7 @@ _GEMINI_TRAFFIC_TYPE_TO_SERVICE_TIER: Final[dict] = {
}
def _map_traffic_type_to_service_tier(traffic_type: str | None) -> str | None:
def _map_traffic_type_to_service_tier(traffic_type: object) -> str | None:
"""
Map a Gemini usageMetadata.trafficType value to a LiteLLM service_tier string.
@ -1273,11 +1282,13 @@ def _served_service_tier(completion_response: object, usage_object: Usage | None
usage_tier: Final = _extract_service_tier(usage_object)
if isinstance(usage_tier, str):
return usage_tier
hidden_params: Final = getattr(completion_response, "_hidden_params", None)
hidden_params: Final[object] = getattr(completion_response, "_hidden_params", None)
if hidden_params is None:
return None
provider_specific: Final = hidden_params.get("provider_specific_fields") or {}
raw_traffic_type: Final = provider_specific.get("traffic_type")
provider_specific: Final[object] = (
require_attributes(hidden_params, _SupportsGet, "get").get("provider_specific_fields") or {}
)
raw_traffic_type: Final = require_attributes(provider_specific, _SupportsGet, "get").get("traffic_type")
if not raw_traffic_type:
return None
return _map_traffic_type_to_service_tier(raw_traffic_type) or "default"
@ -1341,8 +1352,16 @@ def get_usage_object(
return None
class _DumpedUsageFields(TypedDict, total=False, extra_items=object):
pass
def _dumped_usage_fields(usage_model: BaseModel) -> Mapping[str, object]:
return usage_model.model_dump()
def _usage_from_model(usage_model: BaseModel) -> Usage:
return Usage(**usage_model.model_dump())
return Usage(**_DumpedUsageFields(**_dumped_usage_fields(usage_model)))
def _decisions_prompt_tokens_details(usage: DecisionsUsage | OpenAIDecisionUsage) -> PromptTokensDetailsWrapper:

22
litellm/exceptions.py
View file

@ -13,10 +13,11 @@ import datetime
import enum
import sys
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, Protocol, SupportsIndex, SupportsInt, runtime_checkable
import httpx
import openai
from typing_extensions import Buffer
import litellm
from litellm.types.utils import LiteLLMCommonStrings
@ -1277,6 +1278,19 @@ class BlockedPiiEntityError(Exception):
super().__init__(self.message)
@runtime_checkable
class _SupportsTrunc(Protocol):
def __trunc__(self) -> int: ...
def _status_code_int(value: object) -> int:
if isinstance(value, (str, Buffer, SupportsInt, SupportsIndex, _SupportsTrunc)):
return int(value)
raise TypeError(
f"int() argument must be a string, a bytes-like object or a real number, not '{type(value).__name__}'"
)
class MidStreamFallbackError(ServiceUnavailableError):
def __init__(
self,
@ -1292,8 +1306,8 @@ class MidStreamFallbackError(ServiceUnavailableError):
is_pre_first_chunk: bool = False,
emitted_disqualifying_content: bool = False,
):
original_status: Final = getattr(original_exception, "status_code", None)
self.status_code = int(original_status) if original_status is not None else 503
original_status: Final[object] = getattr(original_exception, "status_code", None)
self.status_code = _status_code_int(original_status) if original_status is not None else 503
self.message = f"litellm.MidStreamFallbackError: {message}"
self.model = model
self.llm_provider = llm_provider
@ -1336,7 +1350,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
)
# Restore the propagated status and original response/request objects
self.status_code = int(original_status) if original_status is not None else 503
self.status_code = _status_code_int(original_status) if original_status is not None else 503
self.response = _saved_response
self.request = _saved_request
self.message = _saved_message

47
litellm/harness/endpoint.py
View file

@ -14,10 +14,11 @@ import contextlib
import itertools
import json
import logging
import math
import secrets
from collections.abc import AsyncIterable, AsyncIterator, Mapping
from collections.abc import AsyncIterable, AsyncIterator, Callable, Mapping
from dataclasses import dataclass
from types import MappingProxyType, ModuleType
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Protocol
import httpx
@ -36,6 +37,7 @@ from litellm.constants import (
from litellm.harness.context import GatewayTarget
from litellm.harness.errors import HarnessError, HarnessInstallFailed
from litellm.harness.types import Harness, Usage
from litellm.litellm_core_utils.core_helpers import has_attributes
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
from litellm.types.llms.custom_http import httpxSpecialProvider
@ -45,7 +47,7 @@ if TYPE_CHECKING:
from starlette.requests import Request
from starlette.responses import JSONResponse, Response, StreamingResponse
from starlette.routing import Route
from uvicorn import Server
from uvicorn import Config, Server
verbose_logger: Final = logging.getLogger("LiteLLM")
_JSON_VALUE_ADAPTER: Final = TypeAdapter(object, config=ConfigDict(strict=True))
@ -110,9 +112,22 @@ class _ResponsesModule(Protocol):
StreamingResponse: type[StreamingResponse]
class _UvicornModule(Protocol):
Config: type[Config]
Server: type[Server]
class _CapturesSignals(Protocol):
capture_signals: Callable[[], contextlib.AbstractContextManager[None]]
class _InstallsSignalHandlers(Protocol):
install_signal_handlers: Callable[[], None]
@dataclass(frozen=True)
class _ServerDeps:
uvicorn: ModuleType
uvicorn: _UvicornModule
applications: _ApplicationsModule
routing: _RoutingModule
responses: _ResponsesModule
@ -438,6 +453,14 @@ def _noop() -> None:
return None
def _signal_capture(server: object) -> _CapturesSignals | None:
return server if has_attributes(server, _CapturesSignals, "capture_signals") else None
def _signal_handler_installer(server: object) -> _InstallsSignalHandlers | None:
return server if has_attributes(server, _InstallsSignalHandlers, "install_signal_handlers") else None
class ModelEndpoint:
"""Local HTTP endpoint for one harness session. Use as an async context manager."""
@ -521,7 +544,7 @@ class ModelEndpoint:
await asyncio.sleep(DEFAULT_POLLING_INTERVAL)
def _build_server(self, deps: _ServerDeps) -> Server:
config = deps.uvicorn.Config(
config: Final = deps.uvicorn.Config(
self._build_app(deps),
host=HARNESS_ENDPOINT_HOST,
port=0,
@ -529,14 +552,16 @@ class ModelEndpoint:
log_level="warning",
access_log=False,
lifespan="off",
timeout_graceful_shutdown=HARNESS_PROCESS_KILL_GRACE_SECONDS,
timeout_graceful_shutdown=math.ceil(HARNESS_PROCESS_KILL_GRACE_SECONDS),
)
server = deps.uvicorn.Server(config)
server: Final = deps.uvicorn.Server(config)
# Never touch the host process's signal handlers.
if hasattr(server, "capture_signals"):
server.capture_signals = contextlib.nullcontext
if hasattr(server, "install_signal_handlers"):
server.install_signal_handlers = _noop
signal_capture: Final = _signal_capture(server)
if signal_capture is not None:
signal_capture.capture_signals = contextlib.nullcontext
handler_installer: Final = _signal_handler_installer(server)
if handler_installer is not None:
handler_installer.install_signal_handlers = _noop
return server
def _build_app(self, deps: _ServerDeps) -> Starlette:

21
litellm/images/utils.py
View file

@ -6,12 +6,25 @@ from pathlib import Path
from typing import Any, Final, Protocol, cast, get_type_hints, runtime_checkable
import litellm
from litellm.litellm_core_utils.core_helpers import has_attributes
from litellm.litellm_core_utils.token_counter import get_image_type
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.types.files import FILE_MIME_TYPES, FileType
from litellm.types.images.main import ImageEditOptionalRequestParams
class _ReadsBytes(Protocol):
def read(self, size: int, /) -> bytes: ...
class _Tells(Protocol):
def tell(self) -> int: ...
class _Seeks(Protocol):
def seek(self, offset: int, /) -> object: ...
class ImageEditRequestUtils:
@staticmethod
def get_optional_params_image_edit(
@ -114,12 +127,12 @@ class ImageEditRequestUtils:
bytes_data = image_data[:100]
else:
# For other types, try to read if possible
if hasattr(image_data, "read"):
current_pos = getattr(image_data, "tell", lambda: 0)()
if hasattr(image_data, "seek"):
if has_attributes(image_data, _ReadsBytes, "read"):
current_pos = image_data.tell() if has_attributes(image_data, _Tells, "tell") else 0
if has_attributes(image_data, _Seeks, "seek"):
image_data.seek(0)
bytes_data = image_data.read(100)
if hasattr(image_data, "seek"):
if has_attributes(image_data, _Seeks, "seek"):
image_data.seek(current_pos)
else:
return FILE_MIME_TYPES[FileType.PNG] # Default fallback

4
litellm/integrations/SlackAlerting/user_spend_alerts.py
View file

@ -53,11 +53,13 @@ async def fetch_user_spend_rows(
today: datetime.date,
baseline_days: int,
) -> tuple[UserSpendRow, ...]:
from litellm.proxy.utils import query_raw_rows
today_str: Final = today.strftime("%Y-%m-%d")
month_start_str: Final = today.replace(day=1).strftime("%Y-%m-%d")
baseline_start_str: Final = (today - datetime.timedelta(days=max(baseline_days, 1))).strftime("%Y-%m-%d")
return USER_SPEND_ROWS_ADAPTER.validate_python(
await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str)
await query_raw_rows(prisma_client.db, USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str)
)

23
litellm/integrations/argilla.py
View file

@ -7,11 +7,12 @@ import json
import os
import random
import types
from collections.abc import Mapping
from typing import Final
from collections.abc import Mapping, Sequence
from typing import Final, Protocol
import httpx
from pydantic import BaseModel
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
@ -29,6 +30,22 @@ from litellm.types.integrations.argilla import (
from litellm.types.utils import StandardLoggingPayload
class _ArgillaDataset(TypedDict):
id: ReadOnly[str]
class _ArgillaDatasetsBody(TypedDict, total=False):
items: ReadOnly[Sequence[_ArgillaDataset]]
class _ArgillaDatasetsResponse(Protocol):
def json(self) -> _ArgillaDatasetsBody: ...
def _datasets_body(response: _ArgillaDatasetsResponse) -> _ArgillaDatasetsBody:
return response.json()
def is_serializable(value):
non_serializable_types: Final = (
types.CoroutineType,
@ -102,7 +119,7 @@ class ArgillaLogger(CustomBatchLogger):
url=f"{_credentials_base_url}/api/v1/me/datasets?name={_credentials_dataset_name}",
headers={"X-Argilla-Api-Key": _credentials_api_key},
)
json_response: Final = dataset_response.json()
json_response: Final = _datasets_body(dataset_response)
if (
"items" in json_response
and isinstance(json_response["items"], list)

23
litellm/integrations/arize/_utils.py
View file

@ -1,6 +1,6 @@
import json
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Final, Protocol
from pydantic import ConfigDict, TypeAdapter
from typing_extensions import ReadOnly, TypedDict, override
@ -10,6 +10,7 @@ from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes impor
BaseLLMObsOTELAttributes,
safe_set_attribute,
)
from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes, subscript
from litellm.litellm_core_utils.redact_messages import (
should_redact_message_logging,
)
@ -34,32 +35,40 @@ from litellm.integrations._types.open_inference import (
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
class _SupportsGetWithDefault(Protocol):
def get(self, key: str, default: object = None, /) -> object: ...
class ArizeOTELAttributes(BaseLLMObsOTELAttributes):
@staticmethod
@override
def set_messages(span: "Span", kwargs: dict[str, Any]):
def set_messages(span: "Span", kwargs: Mapping[str, object]):
messages: Final = kwargs.get("messages")
# for /chat/completions
# https://docs.arize.com/arize/large-language-models/tracing/semantic-conventions
if messages:
last_message: Final = messages[-1]
last_message: Final = subscript(messages, -1)
safe_set_attribute(
span,
SpanAttributes.INPUT_VALUE,
last_message.get("content", ""),
require_attributes(last_message, _SupportsGetWithDefault, "get").get("content", ""),
)
# LLM_INPUT_MESSAGES shows up under `input_messages` tab on the span page.
for idx, msg in enumerate(messages):
for idx, msg in enumerate(iter_items(messages)):
prefix = f"{SpanAttributes.LLM_INPUT_MESSAGES}.{idx}"
# Set the role per message.
safe_set_attribute(span, f"{prefix}.{MessageAttributes.MESSAGE_ROLE}", msg.get("role"))
safe_set_attribute(
span,
f"{prefix}.{MessageAttributes.MESSAGE_ROLE}",
require_attributes(msg, _SupportsGetWithDefault, "get").get("role"),
)
# Set the content per message.
safe_set_attribute(
span,
f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}",
msg.get("content", ""),
require_attributes(msg, _SupportsGetWithDefault, "get").get("content", ""),
)
# Additive: emit structured tool_calls / multimodal content

18
litellm/integrations/azure_sentinel/azure_sentinel.py
View file

@ -18,10 +18,11 @@ import time
import traceback
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final, TypeVar
from typing import Final, Protocol, TypeVar
from urllib.parse import urlparse
import httpx
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_logger
from litellm.integrations.batch_utils import (
@ -52,6 +53,19 @@ MONITOR_SCOPE_BY_AUTHORITY_HOST: Final[Mapping[str, str]] = MappingProxyType(
)
class _OAuthTokenBody(TypedDict, total=False):
access_token: ReadOnly[str]
expires_in: ReadOnly[int]
class _OAuthTokenResponse(Protocol):
def json(self) -> _OAuthTokenBody: ...
def _oauth_token_body(response: _OAuthTokenResponse) -> _OAuthTokenBody:
return response.json()
class AzureSentinelLogger(CustomBatchLogger):
"""
Logger that sends LiteLLM logs to Azure Sentinel via Azure Monitor Logs Ingestion API
@ -226,7 +240,7 @@ class AzureSentinelLogger(CustomBatchLogger):
if response.status_code != 200:
raise Exception(f"Failed to get OAuth2 token: {response.status_code} - {response.text}")
token_response: Final = response.json()
token_response: Final = _oauth_token_body(response)
self.oauth_token = token_response.get("access_token")
expires_in: Final = token_response.get("expires_in", 3600)

32
litellm/integrations/bitbucket/bitbucket_client.py
View file

@ -5,7 +5,7 @@ BitBucket API client for fetching .prompt files from BitBucket repositories.
import base64
import urllib.parse
from collections.abc import Mapping
from typing import Final, TypedDict
from typing import Final, Protocol, TypedDict
from typing_extensions import NotRequired, ReadOnly
@ -36,6 +36,30 @@ class BitBucketFileMetadata(TypedDict):
last_modified: ReadOnly[str | None]
class _SrcListingResponse(Protocol):
def json(self) -> BitBucketSrcListing: ...
class _RepositoryInfoResponse(Protocol):
def json(self) -> Mapping[str, object]: ...
class _BranchListingResponse(Protocol):
def json(self) -> BitBucketBranchListing: ...
def _src_listing(response: _SrcListingResponse) -> BitBucketSrcListing:
return response.json()
def _repository_info(response: _RepositoryInfoResponse) -> Mapping[str, object]:
return response.json()
def _branch_listing(response: _BranchListingResponse) -> BitBucketBranchListing:
return response.json()
def _sanitize_file_path(file_path: str) -> str:
"""Reject path traversal and URL-encode each path segment."""
if "#" in file_path or "?" in file_path:
@ -166,7 +190,7 @@ class BitBucketClient:
response: Final = self.http_handler.get(url, headers=self.headers)
response.raise_for_status()
data: Final[BitBucketSrcListing] = response.json()
data: Final = _src_listing(response)
return [
file_path
for item in data.get("values", [])
@ -201,7 +225,7 @@ class BitBucketClient:
try:
response: Final = self.http_handler.get(url, headers=self.headers)
response.raise_for_status()
return response.json()
return _repository_info(response)
except Exception as e:
raise Exception(f"Failed to get repository info: {e}")
@ -231,7 +255,7 @@ class BitBucketClient:
response: Final = self.http_handler.get(url, headers=self.headers)
response.raise_for_status()
data: Final[BitBucketBranchListing] = response.json()
data: Final = _branch_listing(response)
return data.get("values", [])
except Exception as e:
raise Exception(f"Failed to get branches: {e}")

13
litellm/integrations/bitbucket/bitbucket_prompt_manager.py
View file

@ -4,12 +4,13 @@ Fetches .prompt files from BitBucket repositories and provides team-based access
"""
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, Protocol
from jinja2 import DictLoader, select_autoescape
from jinja2.sandbox import ImmutableSandboxedEnvironment
from litellm.integrations.custom_prompt_management import CustomPromptManagement
from litellm.litellm_core_utils.core_helpers import require_attributes
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -26,6 +27,10 @@ from litellm.types.utils import StandardCallbackDynamicParams
from .bitbucket_client import BitBucketClient
class _SupportsGetWithDefault(Protocol):
def get(self, key: str, default: object = None, /) -> object: ...
class BitBucketPromptTemplate:
"""
Represents a prompt template loaded from BitBucket.
@ -35,7 +40,7 @@ class BitBucketPromptTemplate:
self,
template_id: str,
content: str,
metadata: dict[str, Any],
metadata: Mapping[str, object],
model: str | None = None,
):
self.template_id = template_id
@ -44,7 +49,9 @@ class BitBucketPromptTemplate:
self.model = model or metadata.get("model")
self.temperature = metadata.get("temperature")
self.max_tokens = metadata.get("max_tokens")
self.input_schema = metadata.get("input", {}).get("schema", {})
self.input_schema = require_attributes(metadata.get("input", {}), _SupportsGetWithDefault, "get").get(
"schema", {}
)
self.optional_params = {k: v for k, v in metadata.items() if k not in ["model", "input", "content"]}
def __repr__(self):

40
litellm/integrations/clickhouse/clickhouse_spend_logger.py
View file

@ -7,10 +7,10 @@ so `response_id` is always the raw provider response id (cache-hit suffix stripp
import json
import re
from collections.abc import Iterator, Mapping
from collections.abc import Iterable, Iterator, Mapping
from math import isfinite
from types import MappingProxyType
from typing import Any, Final
from typing import Any, Final, Protocol
from pydantic import JsonValue, TypeAdapter, ValidationError
@ -19,6 +19,7 @@ from litellm._logging import verbose_logger
from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger
from litellm.integrations.clickhouse.context import is_lens_analysis
from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
@ -40,6 +41,14 @@ _INTERNAL_METADATA_KEYS: Final = frozenset(
)
class _SupportsGet(Protocol):
def get(self, key: str, /) -> object: ...
class _SupportsItems(Protocol):
def items(self) -> Iterable[tuple[object, object]]: ...
def _request_metadata_fields(value: object) -> Iterator[tuple[str, JsonValue]]:
if value is None:
return
@ -85,22 +94,22 @@ def _json_mapping(value: Mapping[str, object]) -> str:
return _json(dict(value))
def _find_traceparent(metadata: Mapping[str, Any], kwargs: Mapping[str, Any]) -> tuple[str, str]:
def _find_traceparent(metadata: Mapping[str, object], kwargs: Mapping[str, object]) -> tuple[str, str]:
custom_headers = metadata.get("requester_custom_headers") or MappingProxyType({})
proxy_request = (kwargs.get("litellm_params") or MappingProxyType({})).get(
proxy_request = require_attributes(kwargs.get("litellm_params") or MappingProxyType({}), _SupportsGet, "get").get(
"proxy_server_request"
) or MappingProxyType({})
request_headers = proxy_request.get("headers") or MappingProxyType({})
request_headers = require_attributes(proxy_request, _SupportsGet, "get").get("headers") or MappingProxyType({})
for headers in (custom_headers, request_headers):
for name, value in headers.items():
for name, value in require_attributes(headers, _SupportsItems, "items").items():
if str(name).lower() == "traceparent":
return parse_traceparent(value)
return "", ""
def _cache_tokens(usage: Mapping[str, Any]) -> tuple[int, int]:
def _cache_tokens(usage: Mapping[str, object]) -> tuple[int, int]:
"""(cache_read, cache_write) from a Usage dict: OpenAI prompt_tokens_details first, Anthropic fields as fallback."""
details = usage.get("prompt_tokens_details") or MappingProxyType({})
details = require_attributes(usage.get("prompt_tokens_details") or MappingProxyType({}), _SupportsGet, "get")
cache_read = _int(details.get("cached_tokens")) or _int(usage.get("cache_read_input_tokens"))
cache_write = (
_int(details.get("cache_write_tokens"))
@ -116,10 +125,17 @@ def _request_tags(value: object) -> list[str]:
return [str(tag) for tag in value]
def _session_id(payload: StandardLoggingPayload | SpendLogPayload, kwargs: Mapping[str, Any]) -> str:
def _session_id(payload: StandardLoggingPayload | SpendLogPayload, kwargs: Mapping[str, object]) -> str:
"""Mirrors proxy `_get_session_id_for_spend_log`: explicit session id, else the payload trace id."""
request_metadata = (kwargs.get("litellm_params") or MappingProxyType({})).get("metadata") or MappingProxyType({})
return str(payload.get("session_id") or request_metadata.get("session_id") or payload.get("trace_id") or "")
request_metadata = require_attributes(
kwargs.get("litellm_params") or MappingProxyType({}), _SupportsGet, "get"
).get("metadata") or MappingProxyType({})
return str(
payload.get("session_id")
or require_attributes(request_metadata, _SupportsGet, "get").get("session_id")
or payload.get("trace_id")
or ""
)
def _is_trace_ingest(payload: StandardLoggingPayload) -> bool:
@ -128,7 +144,7 @@ def _is_trace_ingest(payload: StandardLoggingPayload) -> bool:
def spend_log_row_from_payload(
payload: StandardLoggingPayload | SpendLogPayload, kwargs: Mapping[str, Any]
payload: StandardLoggingPayload | SpendLogPayload, kwargs: Mapping[str, object]
) -> SpendLogRecord:
metadata: Mapping[str, Any] = payload.get("metadata") or MappingProxyType({})
hidden_params: Mapping[str, Any] = payload.get("hidden_params") or MappingProxyType({})

14
litellm/integrations/custom_guardrail.py
View file

@ -7,7 +7,7 @@ from collections.abc import Iterable, Mapping, Sequence
from datetime import datetime
from itertools import chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args
from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, Protocol, cast, get_args
import httpx
from pydantic import TypeAdapter
@ -20,6 +20,7 @@ from litellm.litellm_core_utils.core_helpers import (
get_metadata_variable_name_from_kwargs,
get_or_create_metadata_bucket,
redact_nested_match_and_regex_keys,
require_attributes,
)
from litellm.secret_managers.main import str_to_bool
from litellm.types.guardrails import (
@ -167,18 +168,22 @@ def _strict_guardrail_modes_enabled() -> bool:
return True if parsed is None else parsed
def get_session_id_from_request_data(request_data: dict[str, Any]) -> str | None:
class _SupportsGet(Protocol):
def get(self, key: str, /) -> object: ...
def get_session_id_from_request_data(request_data: Mapping[str, object]) -> str | None:
"""Extract session_id from request data (litellm_session_id or metadata)."""
session_id = request_data.get("litellm_session_id")
if session_id:
return str(session_id)
metadata: Final = request_data.get("metadata") or {}
metadata: Final = require_attributes(request_data.get("metadata") or {}, _SupportsGet, "get")
session_id = metadata.get("session_id")
if session_id:
return str(session_id)
litellm_metadata: Final = request_data.get("litellm_metadata") or {}
litellm_metadata: Final = require_attributes(request_data.get("litellm_metadata") or {}, _SupportsGet, "get")
session_id = litellm_metadata.get("session_id")
if session_id:
return str(session_id)
@ -291,6 +296,7 @@ def _guardrails_listed_in(metadata: object) -> list[str | dict[str, DynamicGuard
class CustomGuardrail(CustomLogger):
mcp_block_behavior: Literal["tool_result", "raise"] | None = None
# If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path.
use_native_during_call_hook: ClassVar[bool] = False

25
litellm/integrations/email_alerting.py
View file

@ -3,17 +3,29 @@ Functions for sending Email Alerts
"""
import os
from typing import Final
from collections.abc import Awaitable, Mapping, Sequence
from typing import TYPE_CHECKING, Final
from litellm._logging import verbose_logger, verbose_proxy_logger
from litellm.repositories.team_repository import TeamRepository
from litellm.types.integrations.slack_alerting import WebhookEvent
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
# we use this for the email header, please send a test email if you change this. verify it looks good on email
LITELLM_LOGO_URL: Final = "https://litellm-listing.s3.amazonaws.com/litellm_logo.png"
LITELLM_SUPPORT_CONTACT: Final = "support@berri.ai"
def _query_raw_rows(
client: "PrismaClient", query: str, *args: object
) -> Awaitable[Sequence[Mapping[str, object]] | None]:
from litellm.proxy.utils import query_raw_rows
return query_raw_rows(client.db, query, *args)
async def get_all_team_member_emails(team_id: str | None = None) -> list:
verbose_logger.debug("Email Alerting: Getting all team members for team_id=%s", team_id)
if team_id is None:
@ -51,17 +63,18 @@ async def get_all_team_member_emails(team_id: str | None = None) -> list:
WHERE user_id = ANY($1::TEXT[]);
"""
_result: Final = await prisma_client.db.query_raw(sql_query, _team_member_user_ids)
_result: Final = await _query_raw_rows(prisma_client, sql_query, _team_member_user_ids)
verbose_logger.debug("Email Alerting: Got all Emails for team, emails=%s", _result)
if _result is None:
return []
emails: Final = []
for user in _result:
if user and isinstance(user, dict) and user.get("user_email", None) is not None:
emails.append(user.get("user_email"))
emails: Final = [
user.get("user_email")
for user in _result
if user and isinstance(user, dict) and user.get("user_email", None) is not None
]
return emails

4
litellm/integrations/focus/database.py
View file

@ -123,7 +123,9 @@ class FocusLiteLLMDatabase:
ORDER BY ordinal_position;
"""
try:
columns_response: Final = await client.db.query_raw(info_query)
from litellm.proxy.utils import query_raw_rows
columns_response: Final = await query_raw_rows(client.db, info_query)
return {"columns": columns_response, "table_name": "LiteLLM_DailyUserSpend"}
except Exception as exc:
raise RuntimeError(f"Error getting table info: {exc}") from exc

13
litellm/integrations/galileo.py
View file

@ -15,6 +15,7 @@ from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import require_attributes, subscript
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
get_content_from_model_response,
@ -60,6 +61,10 @@ class _JsonResponse(Protocol):
def json(self) -> object: ...
class _SupportsGet(Protocol):
def get(self, key: str, /) -> object: ...
def _login_access_token(response: _GalileoLoginResponse) -> str:
"""Read the bearer token out of a Galileo login response body."""
return response.json()["access_token"]
@ -463,13 +468,13 @@ class GalileoObserve(CustomLogger):
pass
@staticmethod
def _build_prompt(kwargs: Mapping[str, Any]) -> dict[str, object]:
optional_params: Final[Mapping[str, object]] = kwargs.get("optional_params", {}) or {}
def _build_prompt(kwargs: Mapping[str, object]) -> dict[str, object]:
optional_params: Final = require_attributes(kwargs.get("optional_params", {}) or {}, _SupportsGet, "get")
prompt: Final[dict[str, object]] = {"messages": kwargs.get("messages")}
if optional_params.get("functions") is not None:
prompt["functions"] = optional_params["functions"]
prompt["functions"] = subscript(optional_params, "functions")
if optional_params.get("tools") is not None:
prompt["tools"] = optional_params["tools"]
prompt["tools"] = subscript(optional_params, "tools")
return prompt
@staticmethod

14
litellm/integrations/gcs_bucket/gcs_bucket.py
View file

@ -5,7 +5,7 @@ import os
import time
from collections.abc import Mapping
from datetime import datetime, timedelta, timezone
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, Protocol
from litellm._logging import verbose_logger
from litellm._uuid import uuid
@ -16,7 +16,7 @@ from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase
from litellm.litellm_core_utils.cloud_storage_security import (
sanitize_cloud_object_component,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping, require_attributes
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
from litellm.types.integrations.gcs_bucket import *
@ -28,6 +28,10 @@ else:
VertexBase = Any
class _SupportsGetWithDefault(Protocol):
def get(self, key: str, default: str, /) -> object: ...
class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
def __init__(self, bucket_name: str | None = None) -> None:
from litellm.proxy.proxy_server import premium_user
@ -298,7 +302,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
await self._send_individual_logs(items_to_process)
def _get_object_name(
self, kwargs: Mapping[str, object], logging_payload: StandardLoggingPayload, response_obj: Any
self, kwargs: Mapping[str, object], logging_payload: StandardLoggingPayload, response_obj: object
) -> str:
"""
Get the object name to use for the current payload
@ -313,7 +317,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
return f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
if logging_payload.get("error_str", None) is not None:
return self._generate_failure_object_name(request_date_str=current_date)
raw_response_id: Final[object] = response_obj.get("id", "")
raw_response_id: Final[object] = require_attributes(response_obj, _SupportsGetWithDefault, "get").get("id", "")
response_id: Final = raw_response_id if isinstance(raw_response_id, str) else ""
return self._generate_success_object_name(
request_date_str=current_date,
@ -360,7 +364,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
def _generate_success_object_name(
self,
request_date_str: str,
response_id: str,
response_id: object,
) -> str:
return f"{request_date_str}/{response_id}"

2
litellm/integrations/generic_prompt_management/generic_prompt_manager.py
View file

@ -132,7 +132,7 @@ class GenericPromptManager(CustomPromptManagement):
async def async_fetch_prompt_from_api(
self, prompt_id: str | None, prompt_spec: PromptSpec | None
) -> dict[str, Any]:
) -> dict[str, object]:
"""
Fetch a prompt from the API asynchronously.
"""

24
litellm/integrations/humanloop.py
View file

@ -7,10 +7,12 @@ https://humanloop.com/
from typing import Final, cast
import httpx
from pydantic import TypeAdapter
from typing_extensions import TypedDict
import litellm
from litellm.caching import DualCache
from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping, decoded_json_object, subscript
from litellm.llms.custom_httpx.http_handler import get_httpx_client
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import AllMessageValues
@ -19,6 +21,8 @@ from litellm.types.utils import StandardCallbackDynamicParams
from .custom_logger import CustomLogger
_DECODED_JSON: Final = TypeAdapter(object)
class PromptManagementClient(TypedDict):
prompt_id: str
@ -79,18 +83,20 @@ class HumanLoopPromptManager(DualCache):
raise Exception(f"Error getting prompt from Humanloop: {e.response.text}")
json_response: Final = response.json()
template_message: Final = json_response["template"]
if isinstance(template_message, dict):
template_messages = [template_message]
elif isinstance(template_message, list):
template_messages = template_message
decoded_response: Final = _DECODED_JSON.validate_python(json_response)
template_message: Final = subscript(decoded_response, "template")
template_mapping: Final = as_str_mapping(template_message)
template_list: Final = as_object_list(template_message)
if template_mapping is not None:
template_messages = [template_mapping]
elif template_list is not None:
template_messages = template_list
else:
raise ValueError(f"Invalid template message type: {type(template_message)}")
template_model: Final = json_response["model"]
optional_params: Final = {}
for k, v in json_response.items():
if k in litellm.OPENAI_CHAT_COMPLETION_PARAMS:
optional_params[k] = v
optional_params: Final = {
k: v for k, v in decoded_json_object(decoded_response).items() if k in litellm.OPENAI_CHAT_COMPLETION_PARAMS
}
return PromptManagementClient(
prompt_id=humanloop_prompt_id,
prompt_template=cast(list[AllMessageValues], template_messages),

14
litellm/integrations/lago.py
View file

@ -3,7 +3,7 @@
import json
import os
from typing import Final, Literal
from typing import Final, Literal, Protocol
import httpx
@ -11,6 +11,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.llms.custom_httpx.http_handler import (
HTTPHandler,
get_async_httpx_client,
@ -18,6 +19,10 @@ from litellm.llms.custom_httpx.http_handler import (
)
class _SupportsGet(Protocol):
def get(self, key: str, default: object = None, /) -> object: ...
def get_utc_datetime():
import datetime as dt
from datetime import datetime
@ -70,10 +75,11 @@ class LagoLogger(CustomLogger):
if (
isinstance(response_obj, litellm.ModelResponse) or isinstance(response_obj, litellm.EmbeddingResponse)
) and hasattr(response_obj, "usage"):
response_usage: Final = require_attributes(response_obj["usage"], _SupportsGet, "get")
usage = {
"prompt_tokens": response_obj["usage"].get("prompt_tokens", 0),
"completion_tokens": response_obj["usage"].get("completion_tokens", 0),
"total_tokens": response_obj["usage"].get("total_tokens"),
"prompt_tokens": response_usage.get("prompt_tokens", 0),
"completion_tokens": response_usage.get("completion_tokens", 0),
"total_tokens": response_usage.get("total_tokens"),
}
litellm_params: Final = kwargs.get("litellm_params", {}) or {}

5
litellm/integrations/langfuse/langfuse_otel.py
View file

@ -7,6 +7,7 @@ from itertools import chain
from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, runtime_checkable
from opentelemetry import trace
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
@ -33,6 +34,8 @@ from litellm.types.integrations.langfuse_otel import (
)
from litellm.types.utils import StandardCallbackDynamicParams
_DECODED_JSON: Final = TypeAdapter(object)
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
@ -643,7 +646,7 @@ def _tool_call_arguments(arguments: object) -> object:
if not isinstance(arguments, str):
return arguments
try:
return json.loads(arguments)
return _DECODED_JSON.validate_python(json.loads(arguments))
except json.JSONDecodeError:
return {}

13
litellm/integrations/langfuse/langfuse_otel_attributes.py
View file

@ -7,7 +7,7 @@ Relevant Issue: https://github.com/BerriAI/litellm/issues/13764
import json
from collections.abc import Mapping
from datetime import datetime
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Final, Protocol
from pydantic import BaseModel
from typing_extensions import override
@ -16,6 +16,7 @@ from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes impor
BaseLLMObsOTELAttributes,
safe_set_attribute,
)
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse
from litellm.types.utils import (
EmbeddingResponse,
@ -30,6 +31,10 @@ if TYPE_CHECKING:
from opentelemetry.trace import Span
class _SupportsGet(Protocol):
def get(self, key: str, /) -> object: ...
def serialize_langfuse_datetime(value: datetime) -> str:
return json.dumps(value.astimezone().isoformat())
@ -87,11 +92,11 @@ def get_output_content_by_type(
class LangfuseLLMObsOTELAttributes(BaseLLMObsOTELAttributes):
@staticmethod
@override
def set_messages(span: "Span", kwargs: dict[str, Any]):
def set_messages(span: "Span", kwargs: Mapping[str, object]):
prompt: Final = {"messages": kwargs.get("messages")}
optional_params: Final = kwargs.get("optional_params", {})
functions: Final = optional_params.get("functions")
tools: Final = optional_params.get("tools")
functions: Final = require_attributes(optional_params, _SupportsGet, "get").get("functions")
tools: Final = require_attributes(optional_params, _SupportsGet, "get").get("tools")
if functions is not None:
prompt["functions"] = functions
if tools is not None:

25
litellm/integrations/mlflow.py
View file

@ -1,9 +1,11 @@
import json
import threading
from typing import TYPE_CHECKING, Any, Final
from collections.abc import Sequence
from typing import TYPE_CHECKING, Any, Final, Protocol
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import attribute_or_none, iter_items, require_attributes
if TYPE_CHECKING:
from litellm.types.utils import StandardLoggingPayload
@ -11,6 +13,25 @@ else:
StandardLoggingPayload = Any
class _ChoiceWithMessage(Protocol):
@property
def message(self) -> object: ...
class _ExcludeNoneDumpable(Protocol):
def model_dump(self, *, exclude_none: bool) -> object: ...
def _dumped_choice_message(choice: object) -> object:
message: Final = require_attributes(choice, _ChoiceWithMessage, "message").message
return require_attributes(message, _ExcludeNoneDumpable, "model_dump").model_dump(exclude_none=True)
def _output_messages(response_obj: object) -> Sequence[object]:
choices: Final = attribute_or_none(response_obj, "choices") if hasattr(response_obj, "choices") else ()
return [_dumped_choice_message(c) for c in iter_items(choices)]
class MlflowLogger(CustomLogger):
def __init__(self):
from mlflow.tracking import MlflowClient
@ -62,7 +83,7 @@ class MlflowLogger(CustomLogger):
inputs: Final = self._construct_input(kwargs)
input_messages: Final = inputs.get("messages", [])
output_messages: Final = [c.message.model_dump(exclude_none=True) for c in getattr(response_obj, "choices", [])]
output_messages: Final = _output_messages(response_obj)
if messages := [*input_messages, *output_messages]:
set_span_chat_messages(span, messages)
if tools := inputs.get("tools"):

15
litellm/integrations/openmeter.py
View file

@ -4,13 +4,13 @@
import json
import os
from collections.abc import Mapping
from typing import Final
from typing import Final, Protocol
import httpx
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping, require_attributes
from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
from litellm.llms.custom_httpx.http_handler import (
HTTPHandler,
@ -19,6 +19,10 @@ from litellm.llms.custom_httpx.http_handler import (
)
class _SupportsGet(Protocol):
def get(self, key: str, default: object = None, /) -> object: ...
def get_utc_datetime():
import datetime as dt
from datetime import datetime
@ -60,10 +64,11 @@ class OpenMeterLogger(CustomLogger):
if (
isinstance(response_obj, litellm.ModelResponse) or isinstance(response_obj, litellm.EmbeddingResponse)
) and hasattr(response_obj, "usage"):
response_usage: Final = require_attributes(response_obj["usage"], _SupportsGet, "get")
usage = {
"prompt_tokens": response_obj["usage"].get("prompt_tokens", 0),
"completion_tokens": response_obj["usage"].get("completion_tokens", 0),
"total_tokens": response_obj["usage"].get("total_tokens"),
"prompt_tokens": response_usage.get("prompt_tokens", 0),
"completion_tokens": response_usage.get("completion_tokens", 0),
"total_tokens": response_usage.get("total_tokens"),
}
# OPENMETER_TRUST_REQUEST_USER (default "true"): when set to "false",

83
litellm/integrations/opentelemetry.py
View file

@ -6,7 +6,7 @@ from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field
from datetime import datetime
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final, Protocol, cast
from pydantic import ConfigDict, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
@ -35,6 +35,7 @@ from litellm.integrations.otel.model.metadata import flatten_metadata
from litellm.integrations.otel.model.semconv import LiteLLM, Metric
from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls
from litellm.integrations.otel.routing import routing_decision_attributes
from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.secret_redaction import redact_string
@ -47,7 +48,6 @@ from litellm.secret_managers.main import get_secret_bool, str_to_bool
from litellm.types.integrations.otel_span_attributes import SpanAttributes
from litellm.types.services import ServiceLoggerPayload
from litellm.types.utils import (
ChatCompletionMessageToolCall,
CostBreakdown,
Function,
LLMResponseTypes,
@ -130,6 +130,22 @@ def _text_as_stored_on_span(span: Span, key: str, text: str) -> str:
_MAX_DYNAMIC_TRACER_PROVIDERS: Final = 256
class _SupportsGet(Protocol):
def get(self, key: str, /) -> object: ...
def _response_field(response_obj: object, key: str) -> object:
if not response_obj:
return None
return require_attributes(response_obj, _SupportsGet, "get").get(key)
def _billed_usage(response_obj: object, call_type: str | None, litellm_params: Mapping[str, object] | None) -> object:
if not response_obj or is_unbilled_non_inference_call_from_params(call_type, litellm_params, response_obj):
return None
return _response_field(response_obj, "usage")
class _CacheHitView(TypedDict, total=False):
cache_hit: ReadOnly[object]
saved_cache_cost: ReadOnly[object]
@ -2407,17 +2423,17 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
@staticmethod
def _tool_calls_kv_pair(
tool_calls: list[ChatCompletionMessageToolCall],
tool_calls: Iterable[object],
) -> dict[str, object]:
kv_pairs: Final[dict[str, object]] = {}
for idx, tool_call in enumerate(tool_calls):
_function = tool_call.get("function")
_function: object = require_attributes(tool_call, _SupportsGet, "get").get("function")
if not _function:
continue
keys = Function.__annotations__.keys()
for key in keys:
_value = _function.get(key)
_value = require_attributes(_function, _SupportsGet, "get").get(key)
if _value:
kv_pairs[f"{SpanAttributes.LLM_COMPLETIONS.value}.{idx}.function_call.{key}"] = _value
@ -2584,7 +2600,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
# but Embeddings and Image-gen responses do not. Fall back to
# the litellm call ID so every call type can be correlated
# across LiteLLM UI, Phoenix traces, and provider logs (Issue #8).
response_id = (response_obj.get("id") if response_obj else None) or standard_logging_payload.get("id")
response_id = _response_field(response_obj, "id") or standard_logging_payload.get("id")
if response_id:
self.safe_set_attribute(
span=span,
@ -2601,42 +2617,37 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
)
# The model used to generate the response.
if response_obj and response_obj.get("model"):
response_model: Final = _response_field(response_obj, "model")
if response_model:
self.safe_set_attribute(
span=span,
key=SpanAttributes.LLM_RESPONSE_MODEL.value,
value=response_obj.get("model"),
value=response_model,
)
self._set_service_tier_attributes(span=span, standard_logging_payload=standard_logging_payload)
usage: Final = (
response_obj.get("usage")
if response_obj
and not is_unbilled_non_inference_call_from_params(
kwargs.get("call_type"), litellm_params, response_obj
)
else None
)
usage: Final = _billed_usage(response_obj, kwargs.get("call_type"), litellm_params)
if usage:
usage_fields: Final = require_attributes(usage, _SupportsGet, "get")
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS.value,
value=usage.get("total_tokens"),
value=usage_fields.get("total_tokens"),
)
# The number of tokens used in the LLM response (completion).
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS.value,
value=usage.get("completion_tokens"),
value=usage_fields.get("completion_tokens"),
)
# The number of tokens used in the LLM prompt.
self.safe_set_attribute(
span=span,
key=SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS.value,
value=usage.get("prompt_tokens"),
value=usage_fields.get("prompt_tokens"),
)
########################################################################
@ -2712,7 +2723,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
########## LLM Response Attributes ##########
#############################################
if response_obj is not None:
if response_obj.get("choices"):
response_view: Final = require_attributes(response_obj, _SupportsGet, "get")
if response_view.get("choices"):
transformed_choices: Final = self._transform_choices_to_otel_semantic_conventions(
response_obj.get("choices")
)
@ -2722,10 +2734,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
value=safe_dumps(transformed_choices),
)
finish_reasons: Final = []
for idx, choice in enumerate(response_obj.get("choices")):
if choice.get("finish_reason"):
finish_reasons.append(choice.get("finish_reason"))
choice_views: Final = [
require_attributes(choice, _SupportsGet, "get")
for choice in iter_items(response_view.get("choices"))
]
finish_reasons: Final = [
choice.get("finish_reason") for choice in choice_views if choice.get("finish_reason")
]
if finish_reasons:
self.safe_set_attribute(
@ -2734,11 +2749,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
value=safe_dumps(finish_reasons),
)
for idx, choice in enumerate(response_obj.get("choices")):
for choice in choice_views:
if choice.get("finish_reason"):
message = choice.get("message")
if tool_calls := tool_calls_with_server_tool_calls(message):
kv_pairs = OpenTelemetry._tool_calls_kv_pair(list(tool_calls))
tool_calls = tool_calls_with_server_tool_calls(choice.get("message"))
if tool_calls:
kv_pairs = OpenTelemetry._tool_calls_kv_pair(tool_calls)
for key, value in kv_pairs.items():
self.safe_set_attribute(
span=span,
@ -2746,13 +2761,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
value=value,
)
elif response_obj.get("output"):
elif response_view.get("output"):
# Responses API: ResponsesAPIResponse has an "output"
# list instead of "choices". Each item with
# type="message" contains a "content" list of
# OutputText objects (type="output_text").
output_items: Final = response_obj.get("output")
output_messages: Final = self._transform_responses_api_output_to_otel(output_items)
output_items: Final = response_view.get("output")
output_messages: Final = self._transform_responses_api_output_to_otel(iter_items(output_items))
if output_messages:
self.safe_set_attribute(
span=span,
@ -2766,7 +2781,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
# ChatCompletionMessageToolCall format expected by
# _tool_calls_kv_pair.
tool_calls = []
for out_item in output_items:
for out_item in iter_items(output_items):
item_d = self._to_dict(out_item)
if item_d and item_d.get("type") == "function_call":
tool_calls.append(
@ -2787,7 +2802,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
)
# Extract finish reason from ResponsesAPIResponse.status
status: Final = response_obj.get("status")
status: Final = response_view.get("status")
if status:
self.safe_set_attribute(
span=span,
@ -2907,7 +2922,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
return obj.model_dump()
return None
def _transform_responses_api_output_to_otel(self, output: list) -> list[dict]:
def _transform_responses_api_output_to_otel(self, output: Iterable[object]) -> list[dict]:
"""
Transform Responses API output items into OTEL GenAI 1.38 format.

3
litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py
View file

@ -1,4 +1,5 @@
from abc import ABC
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
if TYPE_CHECKING:
@ -7,7 +8,7 @@ if TYPE_CHECKING:
class BaseLLMObsOTELAttributes(ABC):
@staticmethod
def set_messages(span: "Span", kwargs: dict[str, Any]):
def set_messages(span: "Span", kwargs: Mapping[str, object]):
pass
@staticmethod

20
litellm/integrations/otel/plumbing/metrics.py
View file

@ -11,6 +11,7 @@ identical metrics. The attribute cardinality filter is reused from v1 by import
from collections.abc import Mapping
from dataclasses import dataclass
from datetime import datetime
from types import MappingProxyType
from typing import Any, Final, Literal, Protocol, TypeAlias
from opentelemetry.metrics import Histogram, Meter
@ -34,6 +35,7 @@ from litellm.integrations.otel.model.semconv import (
resolve_provider,
)
from litellm.integrations.otel.model.utils import to_seconds
from litellm.litellm_core_utils.core_helpers import attribute_or_none, require_attributes
from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@ -177,7 +179,14 @@ class _MetricKwargs(TypedDict, total=False):
end_time: ReadOnly[datetime | float | str | None]
def resolve_error_type(kwargs: Mapping[str, Any]) -> str:
class _SupportsGet(Protocol):
def get(self, key: str, /) -> object: ...
_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
def resolve_error_type(kwargs: Mapping[str, object]) -> str:
"""The ``error.type`` value for a failed request.
Bounded by construction: the mapped provider exception's class name (the same
@ -188,8 +197,13 @@ def resolve_error_type(kwargs: Mapping[str, Any]) -> str:
span and its exception event, where high cardinality is free.
"""
std_log: Final = kwargs.get("standard_logging_object")
info: Final = getattr(std_log, "error_information", None) or (std_log or {}).get("error_information") or {}
error_class: Final = info.get("error_class") or info.get("error_code")
info: Final = (
attribute_or_none(std_log, "error_information")
or require_attributes(std_log or _NO_FIELDS, _SupportsGet, "get").get("error_information")
or _NO_FIELDS
)
info_fields: Final = require_attributes(info, _SupportsGet, "get")
error_class: Final = info_fields.get("error_class") or info_fields.get("error_code")
if error_class:
return str(error_class)
exception: Final = kwargs.get("exception")

9
litellm/integrations/prompt_layer.py
View file

@ -4,9 +4,12 @@ import os
import traceback
from typing import Final
from pydantic import BaseModel
from pydantic import BaseModel, TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import decoded_json_object
_DECODED_JSON: Final = TypeAdapter(object)
class PromptLayerLogger:
@ -61,8 +64,8 @@ class PromptLayerLogger:
},
)
response_json: Final = request_response.json()
if not request_response.json().get("success", False):
response_json: Final = decoded_json_object(_DECODED_JSON.validate_python(request_response.json()))
if not response_json.get("success", False):
raise Exception("Promptlayer did not successfully log the response!")
print_verbose(f"Prompt Layer Logging: success - final response object: {request_response.text}")

45
litellm/integrations/weave/weave_otel.py
View file

@ -4,7 +4,8 @@ import base64
import json
import os
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Protocol
from opentelemetry.trace import Status, StatusCode
from typing_extensions import override
@ -19,6 +20,7 @@ from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes impor
BaseLLMObsOTELAttributes,
safe_set_attribute,
)
from litellm.litellm_core_utils.core_helpers import has_attributes, require_attributes
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.types.integrations.weave_otel import WeaveOtelConfig, WeaveSpanAttributes
from litellm.types.utils import StandardCallbackDynamicParams
@ -34,6 +36,21 @@ WEAVE_BASE_URL: Final = "https://trace.wandb.ai"
WEAVE_OTEL_ENDPOINT: Final = "/otel/v1/traces"
_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
class _SupportsGet(Protocol):
def get(self, key: str, /) -> object: ...
class _SupportsReplace(Protocol):
def replace(self, old: str, new: str, /) -> object: ...
class _ModelDumpable(Protocol):
def model_dump(self) -> object: ...
class WeaveLLMObsOTELAttributes(BaseLLMObsOTELAttributes):
"""
Weave-specific LLM observability OTEL attributes.
@ -44,15 +61,15 @@ class WeaveLLMObsOTELAttributes(BaseLLMObsOTELAttributes):
@staticmethod
@override
def set_messages(span: Span, kwargs: dict[str, Any]):
def set_messages(span: Span, kwargs: Mapping[str, object]):
"""Set input messages as span attributes using OpenInference conventions."""
messages: Final = kwargs.get("messages") or []
optional_params: Final = kwargs.get("optional_params") or {}
optional_params: Final = kwargs.get("optional_params") or _NO_FIELDS
prompt: Final = {"messages": messages}
functions: Final = optional_params.get("functions")
tools: Final = optional_params.get("tools")
functions: Final = require_attributes(optional_params, _SupportsGet, "get").get("functions")
tools: Final = require_attributes(optional_params, _SupportsGet, "get").get("tools")
if functions is not None:
prompt["functions"] = functions
if tools is not None:
@ -60,7 +77,7 @@ class WeaveLLMObsOTELAttributes(BaseLLMObsOTELAttributes):
safe_set_attribute(span, OpenInferenceSpanAttributes.INPUT_VALUE, json.dumps(prompt))
def _set_weave_specific_attributes(span: Span, kwargs: Mapping[str, Any], response_obj: Any):
def _set_weave_specific_attributes(span: Span, kwargs: Mapping[str, object], response_obj: object):
"""
Sets Weave-specific metadata attributes onto the OTEL span.
@ -69,25 +86,27 @@ def _set_weave_specific_attributes(span: Span, kwargs: Mapping[str, Any], respon
"""
# Extract all needed data upfront
litellm_params: Final = kwargs.get("litellm_params") or {}
litellm_params: Final = kwargs.get("litellm_params") or _NO_FIELDS
# optional_params = kwargs.get("optional_params") or {}
metadata: Final = kwargs.get("metadata") or {}
metadata: Final = kwargs.get("metadata") or _NO_FIELDS
model: Final = kwargs.get("model") or ""
custom_llm_provider: Final = litellm_params.get("custom_llm_provider") or ""
custom_llm_provider: Final = (
require_attributes(litellm_params, _SupportsGet, "get").get("custom_llm_provider") or ""
)
# Weave supports a custom display name and will default to the model name if not provided.
display_name = metadata.get("display_name")
display_name = require_attributes(metadata, _SupportsGet, "get").get("display_name")
if not display_name and model:
if custom_llm_provider:
display_name = f"{custom_llm_provider}/{model}"
else:
display_name = model
if display_name:
display_name = display_name.replace("/", "__")
display_name = require_attributes(display_name, _SupportsReplace, "replace").replace("/", "__")
safe_set_attribute(span, WeaveSpanAttributes.DISPLAY_NAME.value, display_name)
# Weave threads are OpenInference sessions.
if (session_id := metadata.get("session_id")) is not None:
if (session_id := require_attributes(metadata, _SupportsGet, "get").get("session_id")) is not None:
if isinstance(session_id, (list, dict)):
session_id = safe_dumps(session_id)
safe_set_attribute(span, WeaveSpanAttributes.THREAD_ID.value, session_id)
@ -97,7 +116,7 @@ def _set_weave_specific_attributes(span: Span, kwargs: Mapping[str, Any], respon
# but we override them here to better match Weave's expectations
if response_obj:
output_dict = None
if hasattr(response_obj, "model_dump"):
if has_attributes(response_obj, _ModelDumpable, "model_dump"):
output_dict = response_obj.model_dump()
elif hasattr(response_obj, "get"):
output_dict = response_obj

22
litellm/integrations/websearch_interception/tools.py
View file

@ -7,9 +7,19 @@ to this format for consistent interception and execution.
"""
from collections.abc import Mapping
from typing import Any, Final
from typing import Final, Protocol
from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
from litellm.litellm_core_utils.core_helpers import require_attributes
class _SupportsGetWithDefault(Protocol):
def get(self, key: str, default: str, /) -> object: ...
class _SupportsStartswith(Protocol):
def startswith(self, prefix: str, /) -> object: ...
_WEB_SEARCH_TOOL_DESCRIPTION: Final = (
"Search the web for information. Use this when you need current "
@ -164,7 +174,7 @@ def is_chat_completion_request_web_search_tool(tool: Mapping[str, object]) -> bo
return tool.get("type") == "web_search" or is_web_search_tool(dict(tool))
def is_web_search_tool_chat_completion(tool: dict[str, Any]) -> bool:
def is_web_search_tool_chat_completion(tool: Mapping[str, object]) -> bool:
"""
Check if a tool is a web search tool for Chat Completions API (strict check).
@ -197,7 +207,7 @@ def is_web_search_tool_chat_completion(tool: dict[str, Any]) -> bool:
# Check for OpenAI format: {"type": "function", "function": {"name": "litellm_web_search"}}
if tool_type == "function" and "function" in tool:
function_def: Final = tool.get("function", {})
function_name: Final = function_def.get("name", "")
function_name: Final = require_attributes(function_def, _SupportsGetWithDefault, "get").get("name", "")
if function_name == LITELLM_WEB_SEARCH_TOOL_NAME:
return True
@ -229,7 +239,7 @@ def is_anthropic_native_web_search_tool(tool: Mapping[str, object]) -> bool:
return tool_type.startswith("web_search_") and tool_type != "function"
def is_web_search_tool(tool: dict[str, Any]) -> bool:
def is_web_search_tool(tool: Mapping[str, object]) -> bool:
"""
Check if a tool is a web search tool (native or LiteLLM standard).
@ -281,7 +291,7 @@ def is_web_search_tool(tool: dict[str, Any]) -> bool:
# Check for OpenAI format: {"type": "function", "function": {"name": "..."}}
if tool_type == "function" and "function" in tool:
function_def: Final = tool.get("function", {})
function_name: Final = function_def.get("name", "")
function_name: Final = require_attributes(function_def, _SupportsGetWithDefault, "get").get("name", "")
if function_name == LITELLM_WEB_SEARCH_TOOL_NAME:
return True
@ -290,7 +300,7 @@ def is_web_search_tool(tool: dict[str, Any]) -> bool:
return True
# Check for native Anthropic web_search_* types
if tool_type.startswith("web_search_"):
if require_attributes(tool_type, _SupportsStartswith, "startswith").startswith("web_search_"):
return True
# Check for Claude Code's web_search with a type field

12
litellm/integrations/weights_biases.py
View file

@ -1,11 +1,13 @@
from typing import Final
from litellm.litellm_core_utils.core_helpers import iter_items, subscript
imported_openAIResponse = True
try:
import io
import logging
from collections.abc import Mapping
from typing import Any, Literal, Protocol, TypeVar
from typing import Literal, Protocol, TypeVar
from wandb.sdk.data_types import trace_tree
@ -25,7 +27,7 @@ try:
class OpenAIRequestResponseResolver:
def __call__(
self,
request: dict[str, Any],
request: Mapping[str, object],
response: OpenAIResponse,
time_elapsed: float,
) -> trace_tree.WBTraceTree | None:
@ -110,14 +112,14 @@ try:
def _resolve_chat_completion(
self,
request: dict[str, Any],
request: Mapping[str, object],
response: OpenAIResponse,
time_elapsed: float,
) -> trace_tree.WBTraceTree:
"""Resolves the request and response objects for `openai.Completion`."""
prompt: Final = io.StringIO()
for message in request["messages"]:
prompt.write(f"\n\n**{message['role']}**: {message['content']}\n")
for message in iter_items(request["messages"]):
prompt.write(f"\n\n**{subscript(message, 'role')}**: {subscript(message, 'content')}\n")
request_str: Final = prompt.getvalue()
choices: Final = [

12
litellm/interactions/litellm_responses_transformation/transformation.py
View file

@ -8,11 +8,11 @@ This module handles transforming between:
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final, cast
from typing import Final, Protocol, cast
from pydantic import BaseModel
from litellm.litellm_core_utils.core_helpers import billed_usage_cost
from litellm.litellm_core_utils.core_helpers import attribute_or_none, billed_usage_cost, has_attributes
from litellm.types.interactions import (
InteractionInput,
InteractionsAPIOptionalRequestParams,
@ -27,6 +27,10 @@ from litellm.types.llms.openai import (
_STEP_TYPE_ROLES: Final = MappingProxyType({"user_input": "user", "model_output": "assistant"})
class _IsoFormattable(Protocol):
def isoformat(self) -> object: ...
class LiteLLMResponsesInteractionsConfig:
"""Configuration class for transforming between Interactions API and Responses API."""
@ -232,12 +236,12 @@ class LiteLLMResponsesInteractionsConfig:
)
# Convert created_at to ISO string
created_at: Final = getattr(responses_response, "created_at", None)
created_at: Final = attribute_or_none(responses_response, "created_at")
if isinstance(created_at, int):
from datetime import datetime
created = datetime.fromtimestamp(created_at).isoformat()
elif created_at is not None and hasattr(created_at, "isoformat"):
elif created_at is not None and has_attributes(created_at, _IsoFormattable, "isoformat"):
created = created_at.isoformat()
else:
created = None

19
litellm/interactions/streaming_iterator.py
View file

@ -7,8 +7,9 @@ from the Google Interactions API, similar to the responses API streaming iterato
import asyncio
import json
from collections.abc import Mapping
from datetime import datetime
from typing import Any, Final
from typing import Final, Protocol
import httpx
from pydantic import ConfigDict, TypeAdapter
@ -16,7 +17,7 @@ from pydantic import ConfigDict, TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import STREAM_SSE_DONE_STRING
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.core_helpers import process_response_headers, require_attributes
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
from litellm.litellm_core_utils.thread_pool_executor import executor
@ -32,6 +33,10 @@ _INTERACTIONS_STREAMING_CHUNK_ADAPTER: Final = TypeAdapter(
)
class _SupportsGet(Protocol):
def get(self, key: str, default: object, /) -> object: ...
class BaseInteractionsAPIStreamingIterator:
"""
Base class for streaming iterators that process responses from the Interactions API.
@ -45,7 +50,7 @@ class BaseInteractionsAPIStreamingIterator:
model: str | None,
interactions_api_config: BaseInteractionsAPIConfig,
logging_obj: LiteLLMLoggingObj,
litellm_metadata: dict[str, Any] | None = None,
litellm_metadata: Mapping[str, object] | None = None,
custom_llm_provider: str | None = None,
):
self.response = response
@ -65,9 +70,9 @@ class BaseInteractionsAPIStreamingIterator:
model=model or "",
optional_params=self.logging_obj.model_call_details.get("litellm_params", {}),
)
_model_info: Final[dict] = litellm_metadata.get("model_info", {}) if litellm_metadata else {}
_model_info: Final[object] = litellm_metadata.get("model_info", {}) if litellm_metadata else {}
self._hidden_params = {
"model_id": _model_info.get("id", None),
"model_id": require_attributes(_model_info, _SupportsGet, "get").get("id", None),
"api_base": _api_base,
}
self._hidden_params["additional_headers"] = process_response_headers(self.response.headers or {})
@ -135,7 +140,7 @@ class InteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator):
model: str | None,
interactions_api_config: BaseInteractionsAPIConfig,
logging_obj: LiteLLMLoggingObj,
litellm_metadata: dict[str, Any] | None = None,
litellm_metadata: Mapping[str, object] | None = None,
custom_llm_provider: str | None = None,
):
super().__init__(
@ -202,7 +207,7 @@ class SyncInteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator)
model: str | None,
interactions_api_config: BaseInteractionsAPIConfig,
logging_obj: LiteLLMLoggingObj,
litellm_metadata: dict[str, Any] | None = None,
litellm_metadata: Mapping[str, object] | None = None,
custom_llm_provider: str | None = None,
):
super().__init__(

2
litellm/litellm_core_utils/audio_utils/utils.py
View file

@ -313,7 +313,7 @@ def calculate_request_duration(file: FileTypes) -> float | None:
file_content = content
elif hasattr(content, "read") and not isinstance(content, (str, os.PathLike)):
# File-like object in tuple
current_pos: Final = getattr(content, "tell", lambda: None)()
current_pos: Final = content.tell() if hasattr(content, "tell") else None
# Seek to start to ensure we read the entire content
if hasattr(content, "seek"):
content.seek(0)

25
litellm/litellm_core_utils/core_helpers.py
View file

@ -4,6 +4,7 @@ import copy
import logging
import math
import re
import sys
from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
@ -133,7 +134,7 @@ class _SupportsIter(Protocol):
@runtime_checkable
class _SupportsGetItem(Protocol):
def __getitem__(self, index: int, /) -> object: ...
def __getitem__(self, key: str | int, /) -> object: ...
def attribute_or_none(source: object, name: str) -> object:
@ -166,6 +167,28 @@ def iter_items(value: object) -> Iterable[object]:
raise TypeError(f"'{type(value).__name__}' object is not iterable")
@runtime_checkable
class _SupportsContains(Protocol):
def __contains__(self, key: object, /) -> bool: ...
_NOT_A_CONTAINER: Final = "is not a container or iterable" if sys.version_info >= (3, 14) else "is not iterable"
def json_contains(value: object, key: object) -> bool:
if isinstance(value, _SupportsContains):
return key in value
if isinstance(value, (_SupportsIter, _SupportsGetItem)):
return key in tuple(iter_items(value))
raise TypeError(f"argument of type '{type(value).__name__}' {_NOT_A_CONTAINER}")
def subscript(value: object, key: str | int) -> object:
if isinstance(value, _SupportsGetItem):
return value[key]
raise TypeError(f"'{type(value).__name__}' object is not subscriptable")
def attribute_items(source: object, name: str) -> Iterable[object]:
value: Final = attribute_or_none(source, name)
return iter_items(value) if value else ()

17
litellm/litellm_core_utils/dd_tracing.py
View file

@ -5,8 +5,9 @@ If the ddtrace package is not installed, the tracer will be a no-op.
"""
from contextlib import contextmanager
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, Protocol
from litellm.litellm_core_utils.core_helpers import has_attributes
from litellm.secret_managers.main import get_secret_bool
if TYPE_CHECKING:
@ -15,6 +16,14 @@ else:
DD_TRACER = Any
class _TagStrSpan(Protocol):
def set_tag_str(self, key: str, value: str, /) -> object: ...
class _TagSpan(Protocol):
def set_tag(self, key: str, value: str, /) -> object: ...
class NullSpan:
"""A no-op span implementation."""
@ -81,7 +90,7 @@ else:
tracer = NullTracer()
def get_active_span() -> Any | None:
def get_active_span() -> object:
"""
Return the active Datadog span, checking current span first and then root span.
"""
@ -115,10 +124,10 @@ def set_active_span_tag(tag_key: str, tag_value: str) -> bool:
return False
try:
if hasattr(span, "set_tag_str"):
if has_attributes(span, _TagStrSpan, "set_tag_str"):
span.set_tag_str(tag_key, str(tag_value))
return True
if hasattr(span, "set_tag"):
if has_attributes(span, _TagSpan, "set_tag"):
span.set_tag(tag_key, str(tag_value))
return True
except Exception:

93
litellm/litellm_core_utils/litellm_logging.py
View file

@ -19,7 +19,7 @@ from datetime import datetime as dt_object
from enum import Enum
from functools import lru_cache
from types import MappingProxyType, TracebackType
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, TypeVar, cast
from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Protocol, TypeVar, cast
from httpx import Response, Timeout
from openai import AsyncStream as OpenAIAsyncStream
@ -90,6 +90,7 @@ from litellm.litellm_core_utils.classifier_logging import (
is_classifier_call,
)
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
finite_cost,
get_provider_response_headers_from_hidden_params,
is_expected_client_error,
@ -99,6 +100,7 @@ from litellm.litellm_core_utils.core_helpers import (
proxy_stamped_used_client_oauth_token,
reconstruct_model_name,
reports_native_messages_cost,
require_attributes,
set_response_cost_in_hidden_params,
single_log_line,
)
@ -666,6 +668,31 @@ class UsageObservation(Enum):
ESTIMATED = "estimated"
class _A2AUsageSlot(Protocol):
usage: Mapping[str, object] | None
class _A2AResponseCopier(Protocol):
model_copy: Callable[..., _A2AUsageSlot]
class _MetadataLookup(Protocol):
get: Callable[[str], object]
class _ParamsLookup(Protocol):
def get(self, key: str, default: object, /) -> object: ...
def _callback_control_params(litellm_params: object) -> dict[str, object]:
mapping: Final = as_str_mapping(litellm_params)
if isinstance(mapping, dict):
return mapping
if mapping is not None:
return dict(mapping)
return litellm_params.model_dump() if isinstance(litellm_params, BaseModel) else {}
class Logging(LiteLLMLoggingBaseClass):
global \
supabaseClient, \
@ -929,7 +956,9 @@ class Logging(LiteLLMLoggingBaseClass):
new_call.start_time = start_time
new_call.litellm_params = litellm_params
new_call.model_call_details = {
**{key: value for key, value in self.model_call_details.items() if not key.startswith("_accounting_")},
**{
key: value for key, value in self._model_call_details_view.items() if not key.startswith("_accounting_")
},
"litellm_call_id": litellm_call_id,
"start_time": start_time,
"first_api_call_start_time": None,
@ -2467,7 +2496,7 @@ class Logging(LiteLLMLoggingBaseClass):
return self.response_cost_calculator(result=result, cache_hit=cache_hit)
@staticmethod
def is_sync_litellm_request(litellm_params: Mapping[str, object]) -> bool:
def is_sync_litellm_request(litellm_params: _ParamsLookup) -> bool:
"""True for sync SDK entrypoints (``completion``), false for async (``acompletion``, etc.)."""
return (
litellm_params.get(CallTypes.acompletion.value, False) is not True
@ -2918,11 +2947,12 @@ class Logging(LiteLLMLoggingBaseClass):
"""Deprecated alias of mark_logging_complete, kept for callers of the old name"""
self.mark_logging_complete(event_type=event_type)
def should_run_callback(self, callback: litellm.CALLBACK_TYPES, litellm_params: dict, event_hook: str) -> bool:
def should_run_callback(self, callback: litellm.CALLBACK_TYPES, litellm_params: object, event_hook: str) -> bool:
if litellm.global_disable_no_log_param:
return True
if litellm_params.get("no-log", False) is True:
request_params: Final = require_attributes(litellm_params, _ParamsLookup, "get")
if request_params.get("no-log", False) is True:
# proxy cost tracking cal backs should run
if not (isinstance(callback, CustomLogger) and "_PROXY_" in callback.__class__.__name__):
@ -2932,7 +2962,7 @@ class Logging(LiteLLMLoggingBaseClass):
# Check for dynamically disabled callbacks via headers
if EnterpriseCallbackControls is not None and EnterpriseCallbackControls.is_callback_disabled_dynamically(
callback=callback,
litellm_params=litellm_params,
litellm_params=_callback_control_params(litellm_params),
standard_callback_dynamic_params=self.standard_callback_dynamic_params,
):
verbose_logger.debug(
@ -3643,8 +3673,8 @@ class Logging(LiteLLMLoggingBaseClass):
build_logging_payload=True,
publish_logging_payload=not for_async_request,
)
litellm_params = self.model_call_details.get("litellm_params", {})
is_sync_request: Final = self.is_sync_litellm_request(litellm_params)
litellm_params = self._model_call_details_view.get("litellm_params", {})
is_sync_request: Final = self.is_sync_litellm_request(require_attributes(litellm_params, _ParamsLookup, "get"))
try:
## BUILD COMPLETE STREAMED RESPONSE
complete_streaming_response: (
@ -3800,18 +3830,23 @@ class Logging(LiteLLMLoggingBaseClass):
model = call_details_view["model"]
messages = call_details_view["messages"]
optional_params = kwargs.get("optional_params", {})
litellm_params = kwargs.get("litellm_params", {})
optional_params = call_details_view.get("optional_params", {})
litellm_params = call_details_view.get("litellm_params", {})
supabaseClient.log_event(
model=model,
messages=messages,
end_user=optional_params.get("user", "default"),
end_user=require_attributes(optional_params, _ParamsLookup, "get").get("user", "default"),
response_obj=result,
start_time=start_time,
end_time=end_time,
litellm_call_id=(
current_call_id
if (current_call_id := litellm_params.get("litellm_call_id")) is not None
if (
current_call_id := require_attributes(litellm_params, _ParamsLookup, "get").get(
"litellm_call_id", None
)
)
is not None
else str(uuid.uuid4())
),
print_verbose=print_verbose,
@ -4369,7 +4404,7 @@ class Logging(LiteLLMLoggingBaseClass):
for callback in delivery_callbacks:
# check if callback can run for this request
litellm_params = self.model_call_details.get("litellm_params", {})
litellm_params = self._model_call_details_view.get("litellm_params", {})
should_run = self.should_run_callback(
callback=callback,
litellm_params=litellm_params,
@ -4658,8 +4693,8 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug("Logging Details LiteLLM-Failure Call: %s", litellm.failure_callback)
if not self.should_run_logging(event_type="sync_failure"): # prevent double logging
return
litellm_params: Final = self.model_call_details.get("litellm_params", {})
is_sync_request: Final = self.is_sync_litellm_request(litellm_params)
litellm_params: Final = self._model_call_details_view.get("litellm_params", {})
is_sync_request: Final = self.is_sync_litellm_request(require_attributes(litellm_params, _ParamsLookup, "get"))
try:
start_time, end_time = self._failure_handler_helper_fn(
@ -4879,7 +4914,7 @@ class Logging(LiteLLMLoggingBaseClass):
self.mark_logging_complete(event_type="async_failure")
for callback in callbacks:
try:
litellm_params = self.model_call_details.get("litellm_params", {})
litellm_params = self._model_call_details_view.get("litellm_params", {})
should_run = self.should_run_callback(
callback=callback,
litellm_params=litellm_params,
@ -5441,7 +5476,7 @@ class Logging(LiteLLMLoggingBaseClass):
httpx_response: Final[Response | None] = self.model_call_details.get("httpx_response", None)
if httpx_response is None:
raise ValueError("Google GenAI Generate Content: httpx_response is None")
dict_result: Final = httpx_response.json()
dict_result: Final[GenerateContentResponseBody] = httpx_response.json()
result = litellm.VertexGeminiConfig().transform_google_generate_content_to_openai_model_response(
completion_response=dict_result,
model_response=litellm.ModelResponse(),
@ -5454,7 +5489,7 @@ class Logging(LiteLLMLoggingBaseClass):
)
return result
def _handle_a2a_response_logging(self, result: Any) -> object:
def _handle_a2a_response_logging(self, result: object) -> object:
"""
Handles logging for A2A (Agent-to-Agent) responses.
@ -5473,7 +5508,7 @@ class Logging(LiteLLMLoggingBaseClass):
return result
# Deep copy result and add usage
result_copy: Final = result.model_copy(deep=True)
result_copy: Final = require_attributes(result, _A2AResponseCopier, "model_copy").model_copy(deep=True)
result_copy.usage = usage.model_dump() if hasattr(usage, "model_dump") else dict(usage)
return result_copy
@ -7475,7 +7510,7 @@ class StandardLoggingPayloadSetup:
@staticmethod
def get_standard_logging_payload_trace_id(
logging_obj: Logging,
litellm_params: Mapping[str, Any],
litellm_params: Mapping[str, object],
) -> str:
"""
Returns the `litellm_trace_id` for this request
@ -7491,9 +7526,13 @@ class StandardLoggingPayloadSetup:
"""
dynamic_litellm_session_id: Final = litellm_params.get("litellm_session_id")
dynamic_litellm_trace_id: Final = litellm_params.get("litellm_trace_id")
metadata: Final[Mapping[str, object] | None] = litellm_params.get("metadata")
metadata_session_id: Final = metadata.get("session_id") if metadata else None
metadata_trace_id: Final = metadata.get("trace_id") if metadata else None
metadata: Final = litellm_params.get("metadata")
metadata_session_id: Final = (
require_attributes(metadata, _MetadataLookup, "get").get("session_id") if metadata else None
)
metadata_trace_id: Final = (
require_attributes(metadata, _MetadataLookup, "get").get("trace_id") if metadata else None
)
ordered_candidates: Final[tuple[object, object, object, object]] = (
(dynamic_litellm_trace_id, dynamic_litellm_session_id, metadata_trace_id, metadata_session_id)
@ -7508,7 +7547,7 @@ class StandardLoggingPayloadSetup:
@staticmethod
def get_standard_logging_payload_session_id(
logging_obj: Logging,
litellm_params: Mapping[str, Any],
litellm_params: Mapping[str, object],
) -> str:
"""
Returns the end-user/conversation `litellm_session_id` for this request, independent of trace_id.
@ -7523,8 +7562,10 @@ class StandardLoggingPayloadSetup:
dynamic_litellm_session_id: Final[object] = litellm_params.get("litellm_session_id")
if dynamic_litellm_session_id:
return str(dynamic_litellm_session_id)
metadata: Final[Mapping[str, object] | None] = litellm_params.get("metadata")
metadata_session_id: Final = metadata.get("session_id") if metadata else None
metadata: Final = litellm_params.get("metadata")
metadata_session_id: Final = (
require_attributes(metadata, _MetadataLookup, "get").get("session_id") if metadata else None
)
if metadata_session_id:
return str(metadata_session_id)
return logging_obj.litellm_session_id

35
litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py
View file

@ -1,7 +1,8 @@
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Any, Final
from typing import Any, Final, Protocol
from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes
from litellm.types.utils import (
CompletionTokensDetailsWrapper,
PromptTokensDetailsWrapper,
@ -55,19 +56,29 @@ _INTERACTIONS_MODALITY_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
)
def _modality_field(entry: Mapping[str, object]) -> str | None:
return _INTERACTIONS_MODALITY_FIELDS.get(str(entry.get("modality", "")).lower())
class _SupportsGet(Protocol):
def get(self, key: str, default: object = None, /) -> object: ...
def _modality_field(entry: object) -> str | None:
return _INTERACTIONS_MODALITY_FIELDS.get(
str(require_attributes(entry, _SupportsGet, "get").get("modality", "")).lower()
)
def _token_count(value: object) -> int:
return value if isinstance(value, int) else 0
def _modality_token_sums(entries: Sequence[Mapping[str, object]]) -> Mapping[str, int]:
def _modality_token_sums(entries: Sequence[object]) -> Mapping[str, int]:
fields: Final = frozenset(field for entry in entries if (field := _modality_field(entry)) is not None)
return MappingProxyType(
{
field: sum(_token_count(entry.get("tokens")) for entry in entries if _modality_field(entry) == field)
field: sum(
_token_count(require_attributes(entry, _SupportsGet, "get").get("tokens"))
for entry in entries
if _modality_field(entry) == field
)
for field in fields
}
)
@ -119,12 +130,16 @@ class InteractionsUsageObjectTransformation:
return "total_input_tokens" in usage_object or "total_output_tokens" in usage_object
@staticmethod
def transform_interactions_usage_object(usage_object: Mapping[str, Any]) -> Usage:
input_entries: Final = tuple(usage_object.get("input_tokens_by_modality") or ()) + tuple(
usage_object.get("tool_use_tokens_by_modality") or ()
def transform_interactions_usage_object(usage_object: Mapping[str, object]) -> Usage:
input_entries: Final = tuple(iter_items(usage_object.get("input_tokens_by_modality") or ())) + tuple(
iter_items(usage_object.get("tool_use_tokens_by_modality") or ())
)
cached_sums: Final = _modality_token_sums(
tuple(iter_items(usage_object.get("cached_tokens_by_modality") or ()))
)
output_sums: Final = _modality_token_sums(
tuple(iter_items(usage_object.get("output_tokens_by_modality") or ()))
)
cached_sums: Final = _modality_token_sums(tuple(usage_object.get("cached_tokens_by_modality") or ()))
output_sums: Final = _modality_token_sums(tuple(usage_object.get("output_tokens_by_modality") or ()))
total_cached_tokens: Final = _token_count(usage_object.get("total_cached_tokens"))
input_sums: Final = _subtract_cached_from_input(

12
litellm/litellm_core_utils/llm_cost_calc/utils.py
View file

@ -6,10 +6,10 @@ from collections.abc import Mapping, Sequence
from dataclasses import dataclass, field, replace
from datetime import datetime, timezone, tzinfo
from types import MappingProxyType
from typing import Final, Literal, Protocol, TypedDict, cast
from typing import Final, Literal, Protocol, SupportsFloat, SupportsIndex, TypedDict, cast
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from typing_extensions import ReadOnly
from typing_extensions import Buffer, ReadOnly
import litellm
from litellm._internal_context import current_billing_time
@ -48,7 +48,6 @@ _IMAGE_RESPONSE_CALL_TYPES: Final = frozenset(
}
)
# Pre-resolved DataResidency enum values for fast membership checks
_VALID_DATA_RESIDENCIES: Final = frozenset(r.value for r in DataResidency)
_DEPLOYMENT_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields)
@ -1428,6 +1427,13 @@ def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str |
multiplier: Final = model_info.get(f"regional_processing_uplift_multiplier_{residency}")
if multiplier is None:
return 1.0
if not isinstance(multiplier, (str, Buffer, SupportsFloat, SupportsIndex)):
verbose_logger.error(
"Invalid regional_processing_uplift_multiplier_%s for model (got %s); defaulting to 1.0",
residency,
type(multiplier).__name__,
)
return 1.0
try:
return float(multiplier)
except (TypeError, ValueError):

4
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -6,7 +6,7 @@ import json
from collections.abc import Hashable, Iterable, Mapping, Sequence
from dataclasses import dataclass
from itertools import chain, groupby
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
from typing import TYPE_CHECKING, Final, Literal, TypeAlias, cast
from pydantic import TypeAdapter
@ -1143,7 +1143,7 @@ def strip_prefill_prefix_field(message: AllMessageValues) -> AllMessageValues:
return _message_without_fields(message, frozenset({"prefix"}))
def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any:
def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Mapping[object, object]:
"""
Filters a value from a dictionary

33
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -10,7 +10,7 @@ from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Se
from dataclasses import dataclass, replace
from enum import Enum
from types import MappingProxyType
from typing import Annotated, Any, Final, TypeAlias, TypedDict, TypeVar, cast, overload
from typing import Annotated, Any, Final, Protocol, TypeAlias, TypedDict, TypeVar, cast, overload
from jinja2.sandbox import ImmutableSandboxedEnvironment
from pydantic import BaseModel, BeforeValidator, ConfigDict, JsonValue, TypeAdapter, ValidationError, model_validator
@ -22,7 +22,7 @@ import litellm.types.llms
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.constants import REDACTED_BY_LITELLM
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, require_attributes
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
anthropic_system_messages,
find_anthropic_server_tool_result,
@ -4150,7 +4150,7 @@ class BedrockImageProcessor:
_validate_format = validate_format
@staticmethod
def _get_document_format(mime_type: str, supported_doc_formats: list[str]) -> str:
def _get_document_format(mime_type: str, supported_doc_formats: Sequence[str]) -> str:
"""
Get the document format from the mime type
@ -6571,6 +6571,20 @@ def has_tool_with_name(tools: object, tool_name: str) -> bool:
return False
class _SupportsModelDump(Protocol):
def model_dump(self) -> dict[str, object]: ...
class _StructuredMessagesSource(Protocol):
def get_structured_messages(self, data: dict[str, object], /) -> Sequence[object] | None: ...
def _instantiate_structured_message_sources(
source_classes: Iterable[Callable[[], _StructuredMessagesSource]],
) -> Sequence[_StructuredMessagesSource]:
return [source_cls() for source_cls in source_classes]
def resolve_structured_messages(
messages: list[dict[str, object]] | None,
request_kwargs: dict[str, Any],
@ -6608,16 +6622,17 @@ def resolve_structured_messages(
break
# 2. Fallback: try each mapped handler until one produces messages
handlers_to_try: Final[list[Any]] = []
if call_type is not None and call_type in mappings:
handlers_to_try.append(mappings[call_type]())
else:
handlers_to_try.extend(handler_cls() for handler_cls in mappings.values())
handlers_to_try: Final = _instantiate_structured_message_sources(
(mappings[call_type],) if call_type is not None and call_type in mappings else mappings.values()
)
for handler in handlers_to_try:
structured = handler.get_structured_messages(request_kwargs)
if structured:
return [msg if isinstance(msg, dict) else msg.model_dump() for msg in structured]
return [
msg if isinstance(msg, dict) else require_attributes(msg, _SupportsModelDump, "model_dump").model_dump()
for msg in structured
]
return None

6
litellm/litellm_core_utils/prompt_templates/tpm_media_utils.py
View file

@ -202,7 +202,7 @@ def _tpm_embedding_leaf(value: object) -> _TPMMediaProjection:
part: Final = as_str_mapping(value)
if part is not None and part.get("type") == "file":
return _tpm_embedding_file_block(part)
if not isinstance(value, str) or not value.startswith("data:"):
if not isinstance(value, str) or is_plain_embedding_text(value):
return _tpm_media_leaf(value)
if not _is_base64_data_uri(value):
return _TPMMediaProjection(value)
@ -210,6 +210,10 @@ def _tpm_embedding_leaf(value: object) -> _TPMMediaProjection:
return _TPMMediaProjection("" if media else value, media)
def is_plain_embedding_text(value: object) -> bool:
return isinstance(value, str) and not value.startswith("data:")
def _is_json_array(value: object) -> bool:
return isinstance(value, (list, tuple))

316
litellm/litellm_core_utils/realtime_guardrails.py Normal file
View file

@ -0,0 +1,316 @@
import json
from collections.abc import AsyncIterator, Iterable, Mapping, Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final, TypeAlias
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.exceptions import ModifyResponseException
if TYPE_CHECKING:
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.types.guardrails import GuardrailEventHooks
_DEFAULT_VIOLATION_MESSAGE: Final = "I'm sorry, that request was blocked by the content filter."
BLOCKED_TOOL_OUTPUT: Final = json.dumps({"error": "Tool output blocked by content policy"})
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
_JSON_ARRAY: Final = TypeAdapter(tuple[object, ...])
_TEXTS: Final = TypeAdapter(tuple[str, ...])
_TYPED_TEXT_PART_TYPES: Final = frozenset({"input_text", "output_text", "text"})
@dataclass(frozen=True, slots=True)
class RealtimeGuardrailBlock:
message: str
ends_session_on_violation: bool
end_session_after_n_fails: int | None
def ends_session(self, violation_count: int) -> bool:
return self.ends_session_on_violation or (
self.end_session_after_n_fails is not None and violation_count >= self.end_session_after_n_fails
)
@dataclass(frozen=True, slots=True)
class RealtimeGuardrailRewrite:
texts: tuple[str, ...]
RealtimeGuardrailVerdict: TypeAlias = RealtimeGuardrailBlock | RealtimeGuardrailRewrite | None
@dataclass(frozen=True, slots=True)
class RealtimeMessageText:
role: str
texts: tuple[str, ...]
@dataclass(frozen=True, slots=True)
class RealtimeToolOutput:
texts: tuple[str, ...]
sanitized_event: str
RealtimeGuardedItem: TypeAlias = RealtimeMessageText | RealtimeToolOutput
def _json_object(value: object) -> Mapping[str, object] | None:
try:
return _JSON_OBJECT.validate_python(value)
except ValidationError:
return None
def _turn_copy(value: object) -> object:
mapping: Final = _json_object(value)
return value if mapping is None else dict(mapping)
def _typed_text(part: object) -> str | None:
part_object: Final = _json_object(part)
if part_object is None or part_object.get("type") not in _TYPED_TEXT_PART_TYPES:
return None
text: Final[object] = part_object.get("text", "")
return text if isinstance(text, str) else None
def _content_parts(content: object) -> Sequence[object]:
return _JSON_ARRAY.validate_python(content) if isinstance(content, list) else ()
def _typed_texts(content: object) -> tuple[str, ...]:
return tuple(text for text in map(_typed_text, _content_parts(content)) if text is not None)
def _with_text(part: object, text: str | None) -> object:
part_object: Final = _json_object(part)
return part if text is None or part_object is None else {**part_object, "text": text}
def _tool_output(client_event: Mapping[str, object], item: Mapping[str, object]) -> RealtimeToolOutput | None:
output: Final[object] = item.get("output", "")
text: Final = output if isinstance(output, str) else json.dumps(output)
if not text:
return None
sanitized_event: Final = json.dumps({**client_event, "item": {**item, "output": BLOCKED_TOOL_OUTPUT}})
return RealtimeToolOutput(texts=(text,), sanitized_event=sanitized_event)
def realtime_guarded_item(client_event: Mapping[str, object]) -> RealtimeGuardedItem | None:
if client_event.get("type") != "conversation.item.create":
return None
item: Final = _json_object(client_event.get("item"))
if item is None:
return None
if item.get("type") == "function_call_output":
return _tool_output(client_event, item)
texts: Final = _typed_texts(item.get("content"))
role: Final[object] = item.get("role")
return RealtimeMessageText(role=role if isinstance(role, str) else "user", texts=texts) if any(texts) else None
def rewritten_item(client_event: Mapping[str, object], texts: tuple[str, ...]) -> Mapping[str, object]:
item: Final = _json_object(client_event.get("item")) or {}
if item.get("type") == "function_call_output":
return {**item, "output": " ".join(texts)}
parts: Final = _content_parts(item.get("content"))
text_positions: Final = tuple(index for index, part in enumerate(parts) if _typed_text(part) is not None)
replacements: Final = dict(zip(text_positions, texts))
merged_away: Final = frozenset(text_positions[len(texts) :])
return {
**item,
"content": [
_with_text(part, replacements.get(index)) for index, part in enumerate(parts) if index not in merged_away
],
}
def _violation_text(error: Exception) -> str:
detail: Final[object] = getattr(error, "detail", None)
detail_object: Final = _json_object(detail)
if detail_object is not None:
reason: Final[object] = detail_object.get("error")
return reason if isinstance(reason, str) and reason else str(error)
if detail is not None:
return str(detail)
return str(error) or _DEFAULT_VIOLATION_MESSAGE
def _is_guardrail_block(error: Exception) -> bool:
return hasattr(error, "status_code") or isinstance(error, (ValueError, ModifyResponseException))
def _screened_texts(screened: object, texts: tuple[str, ...]) -> tuple[str, ...]:
screened_object: Final = _json_object(screened)
try:
returned: Final = _TEXTS.validate_python(screened_object.get("texts") if screened_object else None)
except ValidationError:
return texts
return returned if len(returned) == len(texts) else texts
async def _guardrail_outcome(
guardrail: "CustomGuardrail", texts: tuple[str, ...], request_data: Mapping[str, object]
) -> RealtimeGuardrailBlock | tuple[str, ...]:
try:
screened: Final = await guardrail.apply_guardrail(
inputs={"texts": list(texts), "images": []},
request_data=dict(request_data),
input_type="request",
)
except Exception as error:
if not _is_guardrail_block(error):
verbose_logger.exception("[realtime guardrail] unexpected error in apply_guardrail: %s", error)
raise
return RealtimeGuardrailBlock(
message=guardrail.realtime_violation_message or _violation_text(error),
ends_session_on_violation=guardrail.on_violation == "end_session",
end_session_after_n_fails=guardrail.end_session_after_n_fails,
)
return _screened_texts(screened, texts)
async def _screen(
guardrails: Sequence["CustomGuardrail"], texts: tuple[str, ...], request_data: Mapping[str, object]
) -> RealtimeGuardrailBlock | tuple[str, ...]:
if not guardrails:
return texts
outcome: Final = await _guardrail_outcome(guardrails[0], texts, request_data)
if isinstance(outcome, RealtimeGuardrailBlock):
return outcome
return await _screen(guardrails[1:], outcome, request_data)
async def _screened_until_block(
guardrails: Sequence["CustomGuardrail"], texts: tuple[str, ...], request_data: Mapping[str, object]
) -> AsyncIterator[RealtimeGuardrailBlock | str]:
for text in texts:
if isinstance(outcome := await _screen(guardrails, (text,), request_data), RealtimeGuardrailBlock):
yield outcome
return
for screened in outcome:
yield screened
async def _screen_each(
guardrails: Sequence["CustomGuardrail"], texts: tuple[str, ...], request_data: Mapping[str, object]
) -> RealtimeGuardrailBlock | tuple[str, ...]:
outcomes: Final = tuple([outcome async for outcome in _screened_until_block(guardrails, texts, request_data)])
block: Final = next((outcome for outcome in outcomes if isinstance(outcome, RealtimeGuardrailBlock)), None)
return block if block is not None else tuple(outcome for outcome in outcomes if isinstance(outcome, str))
def _joins(texts: tuple[str, ...]) -> tuple[str, ...]:
unspaced: Final = "".join(texts)
boundaries: Final = zip(texts, texts[1:])
word_break_unknown: Final = any(not left[-1:].isspace() and not right[:1].isspace() for left, right in boundaries)
return (unspaced, " ".join(texts)) if word_break_unknown else (unspaced,)
async def _rewritten_parts(
guardrails: Sequence["CustomGuardrail"], texts: tuple[str, ...], request_data: Mapping[str, object]
) -> RealtimeGuardrailBlock | tuple[str, ...]:
parts: Final = await _screen_each(guardrails, texts, request_data)
if isinstance(parts, RealtimeGuardrailBlock):
return parts
rejoined: Final = _joins(parts)
screened_rejoined: Final = await _screen_each(guardrails, rejoined, request_data)
if isinstance(screened_rejoined, RealtimeGuardrailBlock):
return screened_rejoined
masked_join: Final = next((text for text, join in zip(screened_rejoined, rejoined) if text != join), None)
return parts if masked_join is None else (masked_join,)
def _in_role_scope(guardrail: "CustomGuardrail", role: str) -> bool:
from litellm.llms.base_llm.guardrail_translation.utils import (
effective_scan_only_tool_results_for_guardrail,
effective_skip_system_message_for_guardrail,
effective_skip_tool_message_for_guardrail,
role_out_of_guardrail_scope,
)
return not role_out_of_guardrail_scope(
role,
skip_system_message=effective_skip_system_message_for_guardrail(guardrail),
skip_tool_message=effective_skip_tool_message_for_guardrail(guardrail),
scan_only_tool_results=effective_scan_only_tool_results_for_guardrail(guardrail),
)
def realtime_runner_guardrail(
callback: object,
check_data: Mapping[str, object],
event_hooks: Sequence["GuardrailEventHooks"],
) -> "CustomGuardrail | None":
from litellm.integrations.custom_guardrail import CustomGuardrail
if not isinstance(callback, CustomGuardrail) or callback.use_native_lifecycle_hooks:
return None
runs: Final = any(
callback.should_run_guardrail(data=check_data, event_type=event_hook) for event_hook in event_hooks
)
return callback if runs else None
def realtime_screening_guardrails(
role: str,
event_hooks: Sequence["GuardrailEventHooks"],
check_data: Mapping[str, object],
callbacks: Iterable[object],
) -> tuple["CustomGuardrail", ...]:
runner_guardrails: Final = (realtime_runner_guardrail(callback, check_data, event_hooks) for callback in callbacks)
guardrails: Final = {
id(guardrail): guardrail
for guardrail in runner_guardrails
if guardrail is not None and _in_role_scope(guardrail, role)
}
return tuple(guardrails.values())
async def realtime_guardrail_verdict(
texts: tuple[str, ...],
role: str,
event_hooks: Sequence["GuardrailEventHooks"],
request_data: Mapping[str, object],
user_api_key_dict: object | None,
callbacks: Iterable[object],
) -> RealtimeGuardrailVerdict:
check_data: Final = {**request_data, "transcript": " ".join(texts), "stream": True}
in_scope: Final = realtime_screening_guardrails(role, event_hooks, check_data, callbacks)
guardrail_request_data: Final = {
**{key: _turn_copy(value) for key, value in request_data.items()},
"user_api_key_dict": user_api_key_dict,
}
joined: Final = _joins(texts)
screened: Final = await _screen_each(in_scope, joined, guardrail_request_data)
if isinstance(screened, RealtimeGuardrailBlock):
return screened
if screened == joined:
return None
rewritten: Final = screened if len(texts) < 2 else await _rewritten_parts(in_scope, texts, guardrail_request_data)
if isinstance(rewritten, RealtimeGuardrailBlock):
return rewritten
return None if rewritten == texts else RealtimeGuardrailRewrite(texts=rewritten)
def guardrail_violation_error_event(message: str) -> str:
return json.dumps(
{
"type": "error",
"error": {"type": "guardrail_violation", "message": message, "code": "content_policy_violation"},
}
)
def guardrail_voice_prompt_events(message: str) -> tuple[str, str]:
prompt: Final = f"Say exactly the following message to the user, word for word, do not add anything else: {message}"
return (
json.dumps(
{
"type": "conversation.item.create",
"item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": prompt}]},
}
),
json.dumps({"type": "response.create"}),
)

296
litellm/litellm_core_utils/realtime_streaming.py
View file

@ -47,13 +47,27 @@ from litellm.types.realtime import ALL_DELTA_TYPES
from .litellm_logging import Logging as LiteLLMLogging
from .realtime_errors import client_close_code, realtime_error_event, websocket_close_reason
from .realtime_guardrails import (
RealtimeGuardedItem,
RealtimeGuardrailBlock,
RealtimeGuardrailRewrite,
RealtimeGuardrailVerdict,
RealtimeMessageText,
RealtimeToolOutput,
guardrail_violation_error_event,
guardrail_voice_prompt_events,
realtime_guarded_item,
realtime_guardrail_verdict,
realtime_runner_guardrail,
realtime_screening_guardrails,
rewritten_item,
)
if TYPE_CHECKING:
from openai.types.realtime.realtime_audio_formats import RealtimeAudioFormats
from websockets.asyncio.client import ClientConnection
from websockets.exceptions import ConnectionClosed
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.types.guardrails import GuardrailEventHooks
CLIENT_CONNECTION_CLASS = ClientConnection
@ -437,21 +451,6 @@ def _opted_out_of_always_on_guardrail(callback: object, request_data: Mapping[st
)
def _realtime_runner_guardrail(
callback: object,
check_data: Mapping[str, object],
event_hooks: Sequence["GuardrailEventHooks"],
) -> "CustomGuardrail | None":
from litellm.integrations.custom_guardrail import CustomGuardrail
if not isinstance(callback, CustomGuardrail) or callback.use_native_lifecycle_hooks:
return None
runs: Final = any(
callback.should_run_guardrail(data=check_data, event_type=event_hook) for event_hook in event_hooks
)
return callback if runs else None
def translation_guardrails_apply(request_data: Mapping[str, object]) -> bool:
from litellm.types.guardrails import GuardrailEventHooks
@ -479,7 +478,7 @@ def realtime_guardrails_apply(request_data: Mapping[str, object], callbacks: Seq
check_data: Final = {**request_data, "stream": True}
runner_event_hooks: Final = (GuardrailEventHooks.realtime_input_transcription, GuardrailEventHooks.pre_call)
return any(
_realtime_runner_guardrail(callback, check_data, runner_event_hooks) is not None for callback in callbacks
realtime_runner_guardrail(callback, check_data, runner_event_hooks) is not None for callback in callbacks
)
@ -1663,16 +1662,60 @@ class RealTimeStreaming:
async def run_realtime_guardrails(
self,
transcript: str,
transcript: object,
item_id: str | None = None,
pre_block_backend_message: str | None = None,
event_hooks: Sequence["GuardrailEventHooks"] | None = None,
) -> bool:
"""
Run registered guardrails on realtime text (transcript, user message, tool output).
Returns True if blocked (synthetic warning already sent to client).
Returns False if clean (caller should send response.create to the backend).
"""
from litellm.types.guardrails import GuardrailEventHooks
if isinstance(transcript, str):
verdict: Final = await self.apply_realtime_guardrails(
(transcript,), "user", item_id, pre_block_backend_message, event_hooks
)
return isinstance(verdict, RealtimeGuardrailBlock)
if realtime_screening_guardrails(
"user",
event_hooks or (GuardrailEventHooks.realtime_input_transcription,),
{**self.request_data, "stream": True},
litellm.callbacks,
):
raise TypeError(f"Realtime guardrails cannot screen a {type(transcript).__name__} transcript")
return False
async def _screen_client_item(self, guarded_item: RealtimeGuardedItem | None) -> RealtimeGuardrailVerdict:
from litellm.types.guardrails import GuardrailEventHooks
match guarded_item:
case RealtimeToolOutput(texts=texts, sanitized_event=sanitized_event):
return await self.apply_realtime_guardrails(
texts,
"tool",
pre_block_backend_message=sanitized_event,
event_hooks=[GuardrailEventHooks.pre_call],
)
case RealtimeMessageText(role=role, texts=texts):
return await self.apply_realtime_guardrails(texts, role, event_hooks=[GuardrailEventHooks.pre_call])
case None:
return None
async def apply_realtime_guardrails(
self,
texts: tuple[str, ...],
role: str,
item_id: str | None = None,
pre_block_backend_message: str | None = None,
event_hooks: Sequence["GuardrailEventHooks"] | None = None,
) -> RealtimeGuardrailVerdict:
"""
Run registered guardrails on realtime text (transcript, user message, tool output).
A block has already been reported to the client and the backend when this returns.
A rewrite carries the masked texts the caller forwards in place of the originals.
``pre_block_backend_message`` (if provided) is sent to the backend
BEFORE any of the guardrail's own backend messages when a block is
@ -1687,102 +1730,33 @@ class RealTimeStreaming:
"""
from litellm.types.guardrails import GuardrailEventHooks
if event_hooks is None:
event_hooks = [GuardrailEventHooks.realtime_input_transcription]
_realtime_event_types: Final = event_hooks
_check_data: Final = {**self.request_data, "transcript": transcript, "stream": True}
_already_run: Final[set] = set()
runner_guardrails: Final = (
_realtime_runner_guardrail(callback, _check_data, _realtime_event_types) for callback in litellm.callbacks
block: Final = await realtime_guardrail_verdict(
texts,
role,
event_hooks if event_hooks is not None else (GuardrailEventHooks.realtime_input_transcription,),
self.request_data,
self.user_api_key_dict,
litellm.callbacks,
)
for callback in runner_guardrails:
if callback is None or id(callback) in _already_run:
continue
_already_run.add(id(callback))
try:
await callback.apply_guardrail(
inputs={"texts": [transcript], "images": []},
request_data={"user_api_key_dict": self.user_api_key_dict},
input_type="request",
)
except Exception as e:
# Re-raise unexpected errors (no status_code/detail = programming bug, not a block).
# HTTPException and guardrail-raised exceptions have a status_code or detail attr.
is_guardrail_block = hasattr(e, "status_code") or isinstance(e, ValueError)
if not is_guardrail_block:
verbose_logger.exception(
"[realtime guardrail] unexpected error in apply_guardrail: %s",
e,
)
raise
# Extract the human-readable error from the detail dict (HTTPException)
# or fall back to str(e) for plain ValueError.
detail: object | None = getattr(e, "detail", None)
if isinstance(detail, dict):
safe_msg = detail.get("error") or str(e)
elif detail is not None:
safe_msg = str(detail)
else:
safe_msg = str(e) or "I'm sorry, that request was blocked by the content filter."
if not isinstance(block, RealtimeGuardrailBlock):
return block
await self._send_blocked_turn_to_backend(pre_block_backend_message, item_id)
await self.websocket.send_text(guardrail_violation_error_event(block.message))
if not self._is_transcription_session:
for voice_prompt_event in guardrail_voice_prompt_events(block.message):
await self._send_to_backend(voice_prompt_event)
# Use realtime_violation_message if configured; fall back to guardrail error text.
error_msg = getattr(callback, "realtime_violation_message", None) or safe_msg
self._violation_count += 1
if block.ends_session(self._violation_count):
verbose_logger.warning("[realtime guardrail] ending session after violation %d", self._violation_count)
await self.backend_ws.close()
await self._send_blocked_turn_to_backend(pre_block_backend_message, item_id)
# Send the policy violation hint (shows as small gray status text in UI).
await self.websocket.send_text(
json.dumps(
{
"type": "error",
"error": {
"type": "guardrail_violation",
"message": error_msg,
"code": "content_policy_violation",
},
}
)
)
if not self._is_transcription_session:
# Ask the LLM to voice the exact guardrail message so the
# user hears it as audio in voice sessions (not just text).
guardrail_prompt = (
f"Say exactly the following message to the user, word for word, "
f"do not add anything else: {error_msg}"
)
await self._send_to_backend(
json.dumps(
{
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": guardrail_prompt}],
},
}
)
)
await self._send_to_backend(json.dumps({"type": "response.create"}))
self._violation_count += 1
end_session_after: int | None = getattr(callback, "end_session_after_n_fails", None)
should_end = getattr(callback, "on_violation", None) == "end_session" or (
end_session_after is not None and self._violation_count >= end_session_after
)
if should_end:
verbose_logger.warning(
"[realtime guardrail] ending session after violation %d",
self._violation_count,
)
await self.backend_ws.close()
verbose_logger.warning(
"[realtime guardrail] BLOCKED transcript (violation %d): %r",
self._violation_count,
transcript[:80],
)
return True
return False
verbose_logger.warning(
"[realtime guardrail] BLOCKED transcript (violation %d): %r",
self._violation_count,
" ".join(texts)[:80],
)
return block
async def _handle_provider_config_message(self, raw_response: str) -> None:
"""Process a backend message when a provider_config is set (transformed path)."""
@ -1866,7 +1840,7 @@ class RealTimeStreaming:
self.store_message(event_str)
self._capture_transcription_usage(event)
await self._send_event_to_client(event, event_str)
blocked = await self.run_realtime_guardrails(cast(str, transcript))
blocked = await self.run_realtime_guardrails(transcript)
if not blocked and self._should_disable_vad_auto_response():
await self._send_to_backend(json.dumps({"type": "response.create"}))
continue
@ -2267,92 +2241,22 @@ class RealTimeStreaming:
guardrail_turn_detection_injected = False
msg_type: str | None = None
try:
from litellm.types.guardrails import GuardrailEventHooks
msg_obj = json.loads(message)
client_event = msg_obj
msg_type = client_event.get("type")
if msg_type == "conversation.item.create":
# Check user text messages for prompt injection
item = client_event.get("item", {})
# Check function_call_output first so a client cannot
# bypass the tool-result guardrail by also setting
# role="user" on a function_call_output item.
if item.get("type") == "function_call_output":
# Tool results are client-controlled and fed to the
# model; check them with the same guardrail used for
# user text so an attacker cannot smuggle blocked
# content into a function_call_output.
output: object = item.get("output", "")
output_text = output if isinstance(output, str) else json.dumps(output)
if output_text:
# Build the sanitized function_call_output up
# front so we can hand it to the guardrail
# runner as the pre-block message. Providers
# that pair every toolCall with a toolResponse
# (e.g. Gemini/Vertex Live) require the
# toolResponse to arrive BEFORE any other
# client message — otherwise the guardrail's
# own clientContent would violate the
# pending-tool-call protocol contract and the
# backend could close the connection before
# the sanitized response ever lands. Dropping
# the blocked item outright would similarly
# leave such providers waiting indefinitely.
# The sanitized payload carries no blocked
# content — only a generic policy marker.
sanitized_msg = json.dumps(
{
**msg_obj,
"item": {
**item,
"output": json.dumps(
{
"error": "Tool output blocked by content policy",
}
),
},
}
)
blocked = await self.run_realtime_guardrails(
output_text,
pre_block_backend_message=sanitized_msg,
event_hooks=[GuardrailEventHooks.pre_call],
)
if blocked:
# ``_pending_guardrail_message`` is
# intentionally NOT set here. That flag
# exists to swallow the reflexive
# ``response.create`` an OpenAI client
# sends immediately after a user text
# message. In a tool-calling flow the
# client may not send a ``response.create``
# at all (e.g. Gemini SDKs auto-respond),
# so leaving the flag set would
# incorrectly drop an unrelated
# ``response.create`` from a later
# interaction turn.
continue
elif item.get("role") == "user":
content_list: Sequence[object] = item.get("content", [])
texts = [
c.get("text", "")
for c in content_list
if isinstance(c, dict) and c.get("type") == "input_text"
]
combined_text = " ".join(texts)
if combined_text:
blocked = await self.run_realtime_guardrails(
combined_text,
event_hooks=[GuardrailEventHooks.pre_call],
)
if blocked:
# Store the guardrail reason so the next response.create
# (sent automatically by the client) is rewritten to
# include it as response instructions.
self._pending_guardrail_message = combined_text
continue # don't forward the original blocked message
self._pending_guardrail_message = None
match await self._screen_client_item(guarded_item := realtime_guarded_item(client_event)):
case RealtimeGuardrailBlock():
if isinstance(guarded_item, RealtimeMessageText):
self._pending_guardrail_message = " ".join(guarded_item.texts)
continue
case RealtimeGuardrailRewrite(texts=masked_texts):
msg_obj["item"] = rewritten_item(client_event, masked_texts)
message = json.dumps(msg_obj)
case None:
pass
if msg_type == "response.create" and self._pending_guardrail_message:
# The guardrail already sent the synthetic AI bubble — drop this

15
litellm/litellm_core_utils/redact_messages.py
View file

@ -12,7 +12,7 @@ import copy
import inspect
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import replace
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, Protocol
from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
@ -21,7 +21,9 @@ from litellm.constants import REDACTED_BY_LITELLM
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.classifier_logging import without_classifier_audit
from litellm.litellm_core_utils.core_helpers import (
attribute_or_none,
get_metadata_variable_name_from_kwargs,
has_attributes,
)
from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY
from litellm.llms.base_llm.guardrail_translation.utils import stream_item_field, stream_item_items
@ -42,6 +44,11 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObject = Any
class _HasArguments(Protocol):
arguments: str
MESSAGE_REDACTION_ENABLE_HEADERS: Final = ("litellm-enable-message-redaction", "x-litellm-enable-message-redaction")
@ -163,7 +170,7 @@ def _redact_bedrock_guardrail_trace(response: object) -> None:
}
response.update(traces)
return
for field, value in ((name, getattr(response, name, None)) for name in BEDROCK_GUARDRAIL_TRACE_FIELDS):
for field, value in ((name, attribute_or_none(response, name)) for name in BEDROCK_GUARDRAIL_TRACE_FIELDS):
if value is not None:
setattr(response, field, _redacted_bedrock_trace(value))
@ -191,8 +198,8 @@ def _redact_tool_calls(tool_calls) -> None:
if not tool_calls:
return
for tool_call in tool_calls:
function = getattr(tool_call, "function", None)
if function is not None and hasattr(function, "arguments"):
function = attribute_or_none(tool_call, "function")
if function is not None and has_attributes(function, _HasArguments, "arguments"):
function.arguments = REDACTED_BY_LITELLM

6
litellm/litellm_core_utils/sensitive_data_masker.py
View file

@ -117,7 +117,9 @@ class SensitiveDataMasker:
elif key_is_sensitive and isinstance(item, str):
masked_items.append(self.mask_value(item))
else:
masked_items.append(item if isinstance(item, (int, float, bool, str, list)) else str(item))
masked_items.append(
item if item is None or isinstance(item, (int, float, bool, str, list)) else str(item)
)
return masked_items
def mask_dict(
@ -144,7 +146,7 @@ class SensitiveDataMasker:
str_value = str(v) if v is not None else ""
masked_data[k] = self.mask_value(str_value)
else:
masked_data[k] = v if isinstance(v, (int, float, bool, str, list)) else str(v)
masked_data[k] = v if v is None or isinstance(v, (int, float, bool, str, list)) else str(v)
except Exception:
masked_data[k] = "<unable to serialize>"

38
litellm/litellm_core_utils/streaming_handler.py
View file

@ -7,7 +7,7 @@ import logging
import threading
import time
import traceback
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence, Sized
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeVar, cast
@ -18,7 +18,7 @@ from openai.types import Completion
from openai.types.chat import ChatCompletionChunk
from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
from openai.types.chat.chat_completion_chunk import ChoiceDelta
from pydantic import BaseModel, ValidationError
from pydantic import BaseModel, TypeAdapter, ValidationError
from typing_extensions import NotRequired, TypedDict
import litellm
@ -28,7 +28,9 @@ from litellm.constants import STREAM_CHUNK_CHOICE_FIELDS, STREAM_FINISH_CHOICE_F
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import (
UPSTREAM_USAGE_COST,
attribute_or_none,
chunk_usage,
decoded_json_object,
provider_reported_usage_cost,
stamp_litellm_bill_on_usage,
)
@ -153,6 +155,13 @@ def _apply_provider_fields(response: ModelResponseStream, fields: Mapping[str, o
setattr(response, key, value)
def _entry_len(entries: Mapping[str, object], key: str) -> int:
value: Final = entries[key]
if isinstance(value, Sized):
return len(value)
raise TypeError(f"object of type '{type(value).__name__}' has no len()")
def _prepend_upstream_choice_extras(
choice: StreamingChoices, upstream_choice: ChunkChoice, extra_keys: frozenset[str]
) -> None:
@ -364,6 +373,9 @@ class _ParsedChunkHiddenParams(LiteLLMBaseModel):
upstream_provider: str | None = None
_DECODED_JSON: Final = TypeAdapter(object)
def _provider_response_model(chunk: object) -> str | None:
model: Final[object] = chunk.get("model") if isinstance(chunk, Mapping) else getattr(chunk, "model", None)
return model if isinstance(model, str) and model else None
@ -450,9 +462,8 @@ class CustomStreamWrapper:
self._closed = False
self._stream_created_time: float = time.time()
litellm_params: Final[GenericLiteLLMParams] = GenericLiteLLMParams.model_validate(
dict(**self.logging_obj.model_call_details.get("litellm_params", {}))
)
raw_litellm_params: Final = dict[str, object](**self.logging_obj.model_call_details.get("litellm_params", {}))
litellm_params: Final[GenericLiteLLMParams] = GenericLiteLLMParams.model_validate(raw_litellm_params)
self.merge_reasoning_content_in_choices: bool = litellm_params.merge_reasoning_content_in_choices or False
self.sent_first_thinking_block = False
self.sent_last_thinking_block = False
@ -485,7 +496,7 @@ class CustomStreamWrapper:
_api_base: Final = get_api_base(
model=model or "",
optional_params=self.logging_obj.model_call_details.get("litellm_params", {}),
optional_params=raw_litellm_params,
)
self._hidden_params: dict[str, object] = {
@ -973,7 +984,7 @@ class CustomStreamWrapper:
if "text_output" in chunk:
response = CustomStreamWrapper.strip_sse_data_from_chunk(chunk) or ""
response = response.strip()
parsed_response = json.loads(response)
parsed_response = decoded_json_object(_DECODED_JSON.validate_python(json.loads(response)))
else:
return {
"text": "",
@ -1107,7 +1118,7 @@ class CustomStreamWrapper:
def is_chunk_non_empty(
self,
completion_obj: dict[str, Any],
completion_obj: Mapping[str, object],
model_response: ModelResponseStream,
response_obj: Mapping[str, object],
) -> bool:
@ -1117,7 +1128,7 @@ class CustomStreamWrapper:
or (
"tool_calls" in completion_obj
and completion_obj["tool_calls"] is not None
and len(completion_obj["tool_calls"]) > 0
and _entry_len(completion_obj, "tool_calls") > 0
)
or ("function_call" in completion_obj and completion_obj["function_call"] is not None)
or (
@ -1876,7 +1887,9 @@ class CustomStreamWrapper:
elif delta.tool_calls is not None:
if isinstance(delta.tool_calls, list):
for t in delta.tool_calls:
if hasattr(t, "functions") and hasattr(getattr(t, "functions"), "arguments"):
if hasattr(t, "functions") and hasattr(
attribute_or_none(t, "functions"), "arguments"
):
if t.function.arguments is None:
t.function.arguments = ""
_json_delta: Final[_DeltaDumpDict] = delta.model_dump()
@ -1914,7 +1927,7 @@ class CustomStreamWrapper:
return
## CHECK FOR TOOL USE
if "tool_calls" in completion_obj and len(completion_obj["tool_calls"]) > 0:
if "tool_calls" in completion_obj and _entry_len(completion_obj, "tool_calls") > 0:
if self.is_function_call is True: # user passed in 'functions' param
completion_obj["function_call"] = completion_obj["tool_calls"][0]["function"]
completion_obj["tool_calls"] = None
@ -2867,8 +2880,9 @@ class CustomStreamWrapper:
def _record_and_strip_usage_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream | None:
self.chunks.append(chunk.model_copy())
dumped_chunk: Final = MappingProxyType[str, object](chunk.model_dump())
usage_free_chunk: Final = self.model_response_creator(
chunk={key: value for key, value in chunk.model_dump().items() if key != "usage"},
chunk={key: value for key, value in dumped_chunk.items() if key != "usage"},
hidden_params=cast( # cast-ok: preserve mapping operations on dynamic storage
Mapping[str, object], getattr(chunk, HIDDEN_PARAMS_ATTR)
),

77
litellm/llms/a2a/chat/guardrail_translation/handler.py
View file

@ -11,13 +11,15 @@ A2A Protocol Format:
"""
import json
from collections.abc import Sequence
from typing import TYPE_CHECKING, Any, Final, Optional
from collections.abc import Mapping, Sequence
from functools import reduce
from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, runtime_checkable
from pydantic import ConfigDict, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import has_attributes, require_attributes, subscript
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
StreamingScanKey,
@ -30,6 +32,38 @@ if TYPE_CHECKING:
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
_JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
_UNSUPPORTED_RESPONSE: Final = object()
class _ModelDumpable(Protocol):
def model_dump(self) -> object: ...
class _ResultLookup(Protocol):
def get(self, key: str, default: object, /) -> object: ...
@runtime_checkable
class _SupportsSetItem(Protocol):
def __setitem__(self, key: str, value: object, /) -> None: ...
def _item_assignment_target(value: object) -> _SupportsSetItem:
if isinstance(value, _SupportsSetItem):
return value
raise TypeError(f"'{type(value).__name__}' object does not support item assignment")
def _response_payload(response: object) -> object:
if has_attributes(response, _ModelDumpable, "model_dump"):
return response.model_dump()
if isinstance(response, dict):
return response
return _UNSUPPORTED_RESPONSE
def _path_step(current: object, key: str) -> object:
return subscript(current, int(key) if key.isdigit() else key)
class _A2ATextPart(TypedDict, total=False):
@ -123,7 +157,7 @@ class A2AGuardrailHandler(BaseTranslation):
async def process_output_response(
self,
response: Any,
response: object,
guardrail_to_apply: "CustomGuardrail",
litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
@ -147,17 +181,12 @@ class A2AGuardrailHandler(BaseTranslation):
Returns:
Modified response with guardrails applied to text content
"""
# Handle both dict and Pydantic model responses
if hasattr(response, "model_dump"):
response_dict = response.model_dump()
is_pydantic = True
elif isinstance(response, dict):
response_dict = response
is_pydantic = False
else:
payload: Final = _response_payload(response)
if payload is _UNSUPPORTED_RESPONSE:
verbose_proxy_logger.warning("A2A: Unknown response type %s, skipping guardrail", type(response))
return response
response_dict: Final = require_attributes(payload, _ResultLookup, "get")
result: Final = response_dict.get("result", {})
if not result or not isinstance(result, dict):
verbose_proxy_logger.debug("A2A: No result in response, skipping guardrail")
@ -218,15 +247,8 @@ class A2AGuardrailHandler(BaseTranslation):
verbose_proxy_logger.debug("A2A: Processed output response")
# Update the original response
if is_pydantic:
# For Pydantic models, we need to update the underlying dict
# and the model will reflect the changes
response_dict["result"] = result
return response
else:
response["result"] = result
return response
_item_assignment_target(response_dict)["result"] = result
return response
async def process_output_streaming_response(
self,
@ -448,20 +470,11 @@ class A2AGuardrailHandler(BaseTranslation):
def _apply_text_to_path(
self,
result: dict[str | int, Any],
result: Mapping[str, object],
path: tuple[str, ...],
part_idx: int,
text: str,
) -> None:
"""Apply guardrailed text back to the specified path in the result."""
# Navigate to the parts list
current = result
for key in path:
if key.isdigit():
# Array index
current = current[int(key)]
else:
current = current[key]
# Update the text in the part
current[part_idx]["text"] = text
parts: Final = reduce(_path_step, path, result)
_item_assignment_target(subscript(parts, part_idx))["text"] = text

9
litellm/llms/anthropic/chat/guardrail_translation/handler.py
View file

@ -795,18 +795,21 @@ class AnthropicMessagesHandler(BaseTranslation):
data: dict, # mutable-ok: the request body the adapter reads
*,
keep_cache_control: bool = False,
keep_tool_errors: bool = False,
) -> ChatCompletionRequest:
"""Translate Anthropic request to OpenAI chat completion format."""
(
chat_completion_compatible_request,
_tool_name_mapping,
) = LiteLLMAnthropicMessagesAdapter(keep_cache_control=keep_cache_control).translate_anthropic_to_openai(
) = LiteLLMAnthropicMessagesAdapter(
keep_cache_control=keep_cache_control, keep_tool_errors=keep_tool_errors
).translate_anthropic_to_openai(
anthropic_message_request=cast(AnthropicMessagesRequest, scannable_tool_result_request(data)),
preserve_midturn_system=True,
)
return chat_completion_compatible_request
def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None:
def get_structured_messages(self, data: dict, *, keep_tool_errors: bool = False) -> list[AllMessageValues] | None:
"""
Convert Anthropic messages request data to OpenAI-spec structured messages.
@ -815,7 +818,7 @@ class AnthropicMessagesHandler(BaseTranslation):
messages: Final = data.get("messages")
if messages is None:
return None
chat_completion_compatible_request: Final = self._translate_to_openai(data)
chat_completion_compatible_request: Final = self._translate_to_openai(data, keep_tool_errors=keep_tool_errors)
result: Final = cast(
list[AllMessageValues],
chat_completion_compatible_request.get("messages", []),

2
litellm/llms/anthropic/chat/handler.py
View file

@ -1081,7 +1081,7 @@ class ModelResponseIterator:
if "caller" in content_block_start["content_block"]:
caller_data: Final = content_block_start["content_block"]["caller"]
if caller_data and tool_use is not None:
tool_use["caller"] = cast(dict[str, Any], caller_data)
tool_use["caller"] = caller_data
elif content_block_start["content_block"]["type"] == "redacted_thinking":
(
thinking_blocks,

4
litellm/llms/anthropic/common_utils.py
View file

@ -2107,7 +2107,7 @@ def _strip_empty_content_blocks_from_message(message: object) -> object:
def strip_empty_content_blocks_from_anthropic_messages(
messages: Sequence[object],
) -> list[Any]:
) -> Sequence[object]:
"""
Return a new message list with empty or whitespace-only ``{"type": "text"}``
content blocks and empty *unsigned* ``{"type": "thinking"}`` blocks removed.
@ -2220,7 +2220,7 @@ def _sanitize_tool_use_id_content_block(block: object) -> object:
return block
def sanitize_tool_use_ids_in_anthropic_messages(messages: Sequence[object]) -> list[Any]:
def sanitize_tool_use_ids_in_anthropic_messages(messages: Sequence[object]) -> Sequence[object]:
"""
Return a new message list with ``tool_use`` / ``server_tool_use`` ``id`` and
``tool_result`` ``tool_use_id`` values rewritten to satisfy Anthropic's

20
litellm/llms/anthropic/messages/adapters/transformation.py
View file

@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm.litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params
from litellm.litellm_core_utils.hidden_params import get_hidden_params
from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_part
from litellm.llms.anthropic.messages.utils import (
@ -434,8 +434,9 @@ _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.va
class LiteLLMAnthropicMessagesAdapter:
def __init__(self, *, keep_cache_control: bool = False) -> None:
def __init__(self, *, keep_cache_control: bool = False, keep_tool_errors: bool = False) -> None:
self.keep_cache_control: Final = keep_cache_control
self.keep_tool_errors: Final = keep_tool_errors
### FOR [BETA] `/v1/messages` endpoint support
@ -615,11 +616,7 @@ class LiteLLMAnthropicMessagesAdapter:
self._add_cache_control_if_applicable(content, doc_obj, model)
new_user_content_list.append(doc_obj)
elif content.get("type") == "tool_result":
tool_result = ChatCompletionToolMessage(
role="tool",
tool_call_id=content.get("tool_use_id", ""),
content=self._tool_result_content(content.get("content")),
)
tool_result = self._tool_message(content)
self._add_cache_control_if_applicable(content, tool_result, model)
tool_message_list.append(tool_result)
@ -1432,6 +1429,15 @@ class LiteLLMAnthropicMessagesAdapter:
return None
return anthropic_image_source_to_openai_url(image_source)
def _tool_message(self, block: object) -> ChatCompletionToolMessage:
fields: Final = as_str_mapping(block) or {}
is_error: Final = fields.get("is_error")
tool_call_id: Final = tool_use_id if isinstance(tool_use_id := fields.get("tool_use_id"), str) else ""
content: Final = self._tool_result_content(fields.get("content"))
if self.keep_tool_errors and isinstance(is_error, bool):
return ChatCompletionToolMessage(role="tool", tool_call_id=tool_call_id, content=content, is_error=is_error)
return ChatCompletionToolMessage(role="tool", tool_call_id=tool_call_id, content=content)
def _tool_result_content(self, raw_content: object) -> ToolResultContent:
if isinstance(raw_content, str):
return raw_content

14
litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py
View file

@ -17,11 +17,14 @@ execute it.
import asyncio
import contextlib
import json
from collections.abc import AsyncIterator
from collections.abc import AsyncIterable, AsyncIterator
from typing import TYPE_CHECKING, Any, Final, cast
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES
from litellm.litellm_core_utils.core_helpers import decoded_json_object, has_attributes
from litellm.types.integrations.custom_logger import AgenticLoopLimitError, AgenticLoopSafetyError
if TYPE_CHECKING:
@ -42,6 +45,7 @@ AGENTIC_LOOP_LIMIT_ERROR_SSE_BYTES: Final = (
b'"The agentic loop reached its limit before this turn finished. '
b'Raise max_agentic_loops to allow more loops."}}\n\n'
)
_DECODED_JSON: Final = TypeAdapter(object)
def is_server_fulfilled_tool_leak_error(chunk: object) -> bool:
@ -76,10 +80,10 @@ def _parse_sse_events(raw: bytes) -> list[tuple]:
continue
data_str = stripped[len("data:") :].strip()
try:
data = json.loads(data_str)
data = _DECODED_JSON.validate_python(json.loads(data_str))
except (json.JSONDecodeError, ValueError):
continue
event_type = current_event_type or data.get("type", "")
event_type = current_event_type or decoded_json_object(data).get("type", "")
current_event_type = None
events.append((event_type, data))
return events
@ -371,7 +375,7 @@ class AgenticAnthropicStreamingIterator:
verbose_logger.debug("AgenticStreamingIterator: Could not rebuild response from SSE bytes")
return
result: Final = await self._http_handler._call_agentic_completion_hooks(
result: Final[object] = await self._http_handler._call_agentic_completion_hooks(
response=rebuilt,
model=self._model,
messages=self._messages,
@ -386,7 +390,7 @@ class AgenticAnthropicStreamingIterator:
if result is None:
return
if hasattr(result, "__aiter__"):
if has_attributes(result, AsyncIterable[object], "__aiter__"):
self._follow_up_iterator = result.__aiter__()
elif isinstance(result, dict):
from litellm.llms.anthropic.messages.native.fake_stream_iterator import (

15
litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py
View file

@ -2,8 +2,8 @@
## Translates OpenAI call to Anthropic `/v1/messages` format
import asyncio
from collections import deque
from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final
from collections.abc import AsyncGenerator, AsyncIterable, Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Final
import anyio
from pydantic import ConfigDict, field_validator
@ -12,6 +12,7 @@ from litellm import verbose_logger
from litellm._logging import redact_internal_details_from_client_message
from litellm._uuid import uuid
from litellm.exceptions import APIError, MidStreamFallbackError
from litellm.litellm_core_utils.core_helpers import has_attributes, iter_items
from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
encrypted_reasoning_signature,
)
@ -126,7 +127,7 @@ class AnthropicResponsesStreamWrapper:
def __init__(
self,
responses_stream: Any,
responses_stream: object,
model: str,
litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
safeguards_evaluator: "SafeguardsEvaluator | None" = None,
@ -414,7 +415,9 @@ class AnthropicResponsesStreamWrapper:
response_obj: Final = getattr(event, "response", None) or (
event.get("response") if isinstance(event, dict) else None
)
output: Final = (getattr(response_obj, "output", None) or ()) if response_obj is not None else ()
output: Final[Iterable[object]] = (
(getattr(response_obj, "output", None) or ()) if response_obj is not None else ()
)
refusal_text: Final = responses_output_refusal_text(output) or (self._refusal_text or None)
status: Final = getattr(response_obj, "status", None) if response_obj is not None else None
has_tool_call: Final = any(
@ -508,14 +511,14 @@ class AnthropicResponsesStreamWrapper:
return self._chunk_queue.popleft()
try:
if hasattr(self.responses_stream, "__aiter__"):
if has_attributes(self.responses_stream, AsyncIterable[object], "__aiter__"):
async for event in self.responses_stream:
self._process_event(event)
if self._chunk_queue:
return self._chunk_queue.popleft()
else:
if self._sync_responses_iterator is None:
self._sync_responses_iterator = iter(self.responses_stream)
self._sync_responses_iterator = iter(iter_items(self.responses_stream))
sync_iterator: Final = self._sync_responses_iterator
missing: Final = object()
while (event := await asyncio.to_thread(next, sync_iterator, missing)) is not missing:

17
litellm/llms/anthropic/messages/responses_adapters/transformation.py
View file

@ -10,6 +10,8 @@ from collections.abc import Iterable, Mapping, Sequence
from itertools import groupby
from typing import Any, Final, cast
from pydantic import TypeAdapter
from litellm.litellm_core_utils.prompt_templates.common_utils import (
TOOL_RESULT_IMAGE_BOUNDARY,
TOOL_RESULT_IMAGE_PLACEHOLDER,
@ -58,6 +60,7 @@ from litellm.types.llms.openai import (
REASONING_SUMMARY_PART_SEPARATOR: Final = "\n\n"
RESPONSES_INCLUDE_ENCRYPTED_REASONING: Final = "reasoning.encrypted_content"
_DECODED_JSON: Final = TypeAdapter(object)
class LiteLLMAnthropicToResponsesAPIAdapter:
@ -805,15 +808,17 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
content.append(reasoning_block)
elif item_type == "function_call":
try:
input_data = json.loads(item.get("arguments", "{}"))
input_data = _DECODED_JSON.validate_python(json.loads(item.get("arguments", "{}")))
except (json.JSONDecodeError, TypeError):
input_data = {}
content.append(
AnthropicResponseContentBlockToolUse(
type="tool_use",
id=item.get("call_id") or item.get("id", ""),
name=item.get("name", ""),
input=input_data,
AnthropicResponseContentBlockToolUse.model_validate(
{
"type": "tool_use",
"id": item.get("call_id") or item.get("id", ""),
"name": item.get("name", ""),
"input": input_data,
}
).model_dump(exclude_none=True)
)
stop_reason = "tool_use"

36
litellm/llms/anthropic/messages/stream_assembly.py
View file

@ -4,10 +4,13 @@ from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping, map_finish_reason
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, map_finish_reason
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import is_str_keyed_dict
from litellm.llms.anthropic.chat.handler import (
ModelResponseIterator as AnthropicModelResponseIterator,
)
@ -21,6 +24,8 @@ from litellm.types.utils import (
TextCompletionResponse,
)
_DECODED_JSON: Final = TypeAdapter(object)
def split_sse_chunk_into_events(chunk: str | bytes) -> list[str]:
"""
@ -98,6 +103,14 @@ _FAST_PATH_DISALLOWED_DELTA_TYPES: Final = frozenset(
)
def _joined_text(parts: Sequence[object]) -> str:
strings: Final = [part for part in parts if isinstance(part, str)]
if len(strings) == len(parts):
return "".join(strings)
first_bad: Final = next(position for position, part in enumerate(parts) if not isinstance(part, str))
raise TypeError(f"sequence item {first_bad}: expected str instance, {type(parts[first_bad]).__name__} found")
def collapse_pure_text_chunks(
all_chunks: Sequence[str | bytes],
) -> list[str] | None:
@ -120,10 +133,10 @@ def collapse_pure_text_chunks(
if ev:
normalized.append(ev)
text_block_indexes: Final[set] = set()
text_block_indexes: Final[set[object]] = set()
out: Final[list[str]] = []
pending_text: list[str] = []
pending_index: int | None = None
pending_text: list[object] = []
pending_index: object = None
saw_any_text_delta = False
def flush() -> None:
@ -132,7 +145,7 @@ def collapse_pure_text_chunks(
merged: Final = {
"type": "content_block_delta",
"index": pending_index if pending_index is not None else 0,
"delta": {"type": "text_delta", "text": "".join(pending_text)},
"delta": {"type": "text_delta", "text": _joined_text(pending_text)},
}
out.append("data: " + json.dumps(merged))
pending_text = []
@ -158,20 +171,20 @@ def collapse_pure_text_chunks(
if not json_str.startswith("{"):
continue
try:
data = json.loads(json_str)
data = decoded_json_object(_DECODED_JSON.validate_python(json.loads(json_str)))
except (json.JSONDecodeError, ValueError):
return None
etype = data.get("type")
if etype == "content_block_start":
block = data.get("content_block") or {}
block = decoded_json_object(data.get("content_block") or {})
if block.get("type") != "text":
return None
text_block_indexes.add(data.get("index"))
flush()
out.append(ev)
elif etype == "content_block_delta":
delta = data.get("delta") or {}
delta = decoded_json_object(data.get("delta") or {})
dtype = delta.get("type")
if dtype in _FAST_PATH_DISALLOWED_DELTA_TYPES:
return None
@ -291,9 +304,10 @@ def extract_sse_data(event_str: str) -> dict | None:
if payload is None or not payload.startswith("{"):
return None
try:
return json.loads(payload)
decoded: Final = _DECODED_JSON.validate_python(json.loads(payload))
except (ValueError, TypeError):
return None
return decoded if is_str_keyed_dict(decoded) else None
def build_usage_only_response_from_chunks(
@ -492,7 +506,7 @@ def _carries_no_data(fields: Sequence[tuple[str, str]]) -> bool:
def _json_mapping(payload: str) -> Mapping[str, object] | None:
try:
value: Final[object] = json.loads(payload)
value: Final = _DECODED_JSON.validate_python(json.loads(payload))
except ValueError:
return None
return as_str_mapping(value)
@ -555,7 +569,7 @@ def is_anthropic_sse_stream(all_chunks: Sequence[object]) -> bool:
def _is_json_no_reader_can_take(payload: str) -> bool:
try:
value: Final[object] = json.loads(payload)
value: Final = _DECODED_JSON.validate_python(json.loads(payload))
except ValueError:
return False
if value is None:

29
litellm/llms/apiserpent/search/transformation.py
View file

@ -12,7 +12,9 @@ from typing import Final, Literal, cast
from urllib.parse import urlencode
import httpx
from pydantic import TypeAdapter
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, iter_items
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.apiserpent.search.defaults import (
@ -33,6 +35,7 @@ from litellm.secret_managers.main import get_secret_str
DEEP_SEARCH_PARAM: Final = "deep"
APISERPENT_BASE: Final = "https://apiserpent.com"
APISERPENT_PARAMS_KEY: Final = "_apiserpent_params"
_DECODED_JSON: Final = TypeAdapter(object)
class APISerpentSearchConfig(BaseSearchConfig):
@ -150,22 +153,24 @@ class APISerpentSearchConfig(BaseSearchConfig):
Full format nests results under ``results.organic[]``; simple format
returns a flat ``results[]`` array. Both expose title/url/snippet.
"""
response_json: Final = raw_response.json()
response_json: Final = decoded_json_object(_DECODED_JSON.validate_python(raw_response.json()))
raw_results: Final = response_json.get("results") or {}
organic: Final = raw_results.get("organic", []) if isinstance(raw_results, dict) else raw_results
nested_results: Final = as_str_mapping(raw_results)
organic: Final = nested_results.get("organic", []) if nested_results is not None else raw_results
results: Final[list[SearchResult]] = []
for result in organic:
results.append(
SearchResult(
title=result.get("title", ""),
url=result.get("url", ""),
snippet=result.get("snippet", ""),
date=result.get("date"),
last_updated=None,
)
results: Final = [
SearchResult.model_validate(
{
"title": result.get("title", ""),
"url": result.get("url", ""),
"snippet": result.get("snippet", ""),
"date": result.get("date"),
"last_updated": None,
}
)
for result in map(decoded_json_object, iter_items(organic))
]
return SearchResponse(
results=results,

12
litellm/llms/azure/azure.py
View file

@ -2,7 +2,7 @@ import asyncio
import json
import time
from collections.abc import Callable, Coroutine, Mapping
from typing import Final
from typing import Final, Protocol
import httpx
from openai import (
@ -17,6 +17,7 @@ from pydantic import BaseModel, ConfigDict
import litellm
from litellm.constants import DEFAULT_MAX_RETRIES
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_utils import speech_request_body, track_llm_api_timing
from litellm.llms.custom_httpx.http_handler import (
@ -61,6 +62,11 @@ from .image_generation import (
from .image_generation.http_utils import azure_deployment_image_generation_json_body
class _HasResponseText(Protocol):
@property
def text(self) -> str: ...
class AzureOpenAIAssistantsAPIConfig:
"""
Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/assistants-reference-messages?tabs=python#create-message
@ -864,11 +870,11 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
except Exception as e:
status_code: Final = getattr(e, "status_code", 500)
error_headers = getattr(e, "headers", None)
error_response: Final = getattr(e, "response", None)
error_response: Final[object] = getattr(e, "response", None)
error_text = str(e)
if error_headers is None and error_response:
error_headers = getattr(error_response, "headers", None)
error_text = error_response.text
error_text = require_attributes(error_response, _HasResponseText, "text").text
raise AzureOpenAIError(
status_code=status_code,
message=error_text,

5
litellm/llms/azure/responses/transformation.py
View file

@ -251,9 +251,8 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
def compact_url(self, api_base: str) -> str:
params: Final = httpx.URL(api_base).params
v1_params: Final = (
params if BaseAzureLLM.is_azure_v1_api_version(params.get("api-version")) else params.remove("api-version")
)
api_version: Final = next(iter(params.get_list("api-version")), None)
v1_params: Final = params if BaseAzureLLM.is_azure_v1_api_version(api_version) else params.remove("api-version")
return super().compact_url(str(_v1_responses_url(api_base).copy_with(params=v1_params)))
#########################################################

6
litellm/llms/azure_ai/agents/handler.py
View file

@ -308,7 +308,7 @@ class AzureAIAgentsHandler:
logging_obj: LiteLLMLoggingObj,
optional_params: dict,
litellm_params: dict,
timeout: float,
timeout: float | str | httpx.Timeout | None,
client: HTTPHandler | None = None,
headers: dict | None = None,
) -> ModelResponse:
@ -432,7 +432,7 @@ class AzureAIAgentsHandler:
logging_obj: LiteLLMLoggingObj,
optional_params: dict,
litellm_params: dict,
timeout: float,
timeout: float | str | httpx.Timeout | None,
client: AsyncHTTPHandler | None = None,
headers: dict | None = None,
) -> ModelResponse:
@ -559,7 +559,7 @@ class AzureAIAgentsHandler:
logging_obj: LiteLLMLoggingObj,
optional_params: dict,
litellm_params: dict,
timeout: float,
timeout: float | str | httpx.Timeout | None,
headers: dict | None = None,
) -> AsyncIterator[ModelResponseStream]:
"""Execute async streaming completion using Azure Agent Service with native SSE."""

2
litellm/llms/azure_ai/agents/transformation.py
View file

@ -323,7 +323,7 @@ class AzureAIAgentsConfig(BaseConfig):
logging_obj: LiteLLMLoggingObj,
optional_params: dict,
litellm_params: dict,
timeout: float | Any,
timeout: float | str | httpx.Timeout | None,
acompletion: bool,
stream: bool | None = False,
headers: dict | None = None,

8
litellm/llms/azure_ai/anthropic/count_tokens/handler.py
View file

@ -8,7 +8,7 @@ from collections.abc import Mapping
from typing import Final
import httpx
from pydantic import JsonValue
from pydantic import JsonValue, TypeAdapter
import litellm
from litellm._logging import verbose_logger
@ -22,6 +22,8 @@ from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
)
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
_DECODED_JSON: Final = TypeAdapter(object)
class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
"""
@ -42,7 +44,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
system: object = None,
optional_params: Mapping[str, JsonValue] | None = None,
context_management: JsonValue = None,
) -> dict[str, object]:
) -> object:
"""
Handle a CountTokens request using httpx with Azure authentication.
@ -126,7 +128,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
message=error_text,
)
azure_response: Final = response.json()
azure_response: Final = _DECODED_JSON.validate_python(response.json())
verbose_logger.debug("Azure AI Anthropic response: %s", azure_response)

6
litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py
View file

@ -11,6 +11,7 @@ from pydantic import JsonValue
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import decoded_json_object
from litellm.llms.anthropic.count_tokens.transformation import (
AnthropicCountFields,
ChatCountTranslationError,
@ -139,13 +140,14 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
)
if result is not None:
response_fields: Final = decoded_json_object(result)
return TokenCountResponse.model_validate(
{
"total_tokens": result.get("input_tokens", 0),
"total_tokens": response_fields.get("input_tokens", 0),
"request_model": request_model,
"model_used": model_to_use,
"tokenizer_type": "azure_ai_anthropic_api",
"original_response": result,
"original_response": response_fields,
}
)
except AnthropicError as e:

10
litellm/llms/azure_ai/anthropic/messages_transformation.py
View file

@ -3,7 +3,7 @@ Azure Anthropic messages transformation config - extends AnthropicMessagesConfig
"""
from collections.abc import Sequence
from typing import Any, Final
from typing import Final
from litellm.llms.anthropic.messages.native.transformation import (
AnthropicMessagesConfig,
@ -109,14 +109,10 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig):
Processes both `system` and `messages` content blocks.
"""
def _sanitize(cache_control: Any) -> None:
if isinstance(cache_control, dict):
cache_control.pop("scope", None)
def _process_content_list(content: list) -> None:
for item in content:
if isinstance(item, dict) and "cache_control" in item:
_sanitize(item["cache_control"])
if isinstance(item, dict) and "cache_control" in item and isinstance(item["cache_control"], dict):
item["cache_control"].pop("scope", None)
if "system" in anthropic_messages_request:
system: Final = anthropic_messages_request["system"]

13
litellm/llms/azure_ai/image_edit/flux2_transformation.py
View file

@ -2,11 +2,12 @@ import base64
from collections.abc import Mapping, Sequence
from io import BufferedReader
from types import MappingProxyType
from typing import Any, Final
from typing import Final, Protocol
from httpx._types import RequestFiles
import litellm
from litellm.litellm_core_utils.core_helpers import has_attributes
from litellm.llms.azure_ai.common_utils import (
AzureFoundryModelInfo,
get_azure_ai_auth_headers,
@ -21,6 +22,10 @@ from litellm.types.llms.openai import FileTypes
from litellm.types.router import GenericLiteLLMParams
class _BinaryReadable(Protocol):
def read(self) -> bytes: ...
class AzureFoundryFlux2ImageEditConfig(OpenAIImageEditConfig):
"""
Azure AI Foundry FLUX 2 image edit config
@ -100,7 +105,7 @@ class AzureFoundryFlux2ImageEditConfig(OpenAIImageEditConfig):
if image is None:
raise ValueError("FLUX 2 image edit requires an image.")
images: Final = tuple(image) if isinstance(image, list) else (image,)
images: Final[Sequence[object]] = [*image] if isinstance(image, list) else [image]
if not images:
raise ValueError("FLUX 2 image edit requires at least one image.")
max_reference_images: Final = 10 if "flex" in model.lower() else 8
@ -121,14 +126,14 @@ class AzureFoundryFlux2ImageEditConfig(OpenAIImageEditConfig):
}
return request_body, []
def _convert_image_to_base64(self, image: Any) -> str:
def _convert_image_to_base64(self, image: object) -> str:
"""Convert image file to base64 string"""
if isinstance(image, BufferedReader):
image_bytes = image.read()
image.seek(0) # Reset file pointer for potential reuse
elif isinstance(image, bytes):
image_bytes = image
elif hasattr(image, "read"):
elif has_attributes(image, _BinaryReadable, "read"):
image_bytes = image.read()
else:
raise ValueError(f"Unsupported image type: {type(image)}")

21
litellm/llms/base_llm/guardrail_translation/base_translation.py
View file

@ -1,7 +1,9 @@
from abc import ABC, abstractmethod
from collections.abc import Sequence
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional
from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional, Protocol
from litellm.litellm_core_utils.core_helpers import has_attributes
if TYPE_CHECKING:
from fastapi import HTTPException
@ -15,6 +17,10 @@ if TYPE_CHECKING:
from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
class _ModelDumpable(Protocol):
def model_dump(self) -> object: ...
@dataclass(slots=True)
class StreamTransformSink:
"""Out-parameter used by ``process_output_streaming_response`` to hand the
@ -81,7 +87,7 @@ class BaseTranslation(ABC):
@staticmethod
def transform_user_api_key_dict_to_metadata(
user_api_key_dict: Any | None,
user_api_key_dict: object,
) -> dict[str, object]:
"""
Transform user_api_key_dict to a metadata dict with prefixed keys.
@ -99,7 +105,11 @@ class BaseTranslation(ABC):
return {}
# Convert to dict if it's a Pydantic object
user_dict = user_api_key_dict.model_dump() if hasattr(user_api_key_dict, "model_dump") else user_api_key_dict
user_dict: Final = (
user_api_key_dict.model_dump()
if has_attributes(user_api_key_dict, _ModelDumpable, "model_dump")
else user_api_key_dict
)
if not isinstance(user_dict, dict):
return {}
@ -257,11 +267,12 @@ class BaseTranslation(ABC):
"""
return None
def get_structured_messages(self, data: dict) -> list["AllMessageValues"] | None:
def get_structured_messages(self, data: dict, *, keep_tool_errors: bool = False) -> list["AllMessageValues"] | None:
"""
Convert request data to OpenAI-spec structured messages.
Override in subclasses for format-specific conversion.
Override in subclasses for format-specific conversion. ``keep_tool_errors`` copies a provider's
tool-failure marker onto tool messages as ``is_error``; only for copies that are never sent to a model.
Returns None if no convertible content is found.
"""

14
litellm/llms/base_llm/managed_resources/isolation.py
View file

@ -11,8 +11,9 @@ are denied so an empty user_id can never select an unscoped query.
"""
from collections.abc import Mapping, Sequence
from typing import Any, Final
from typing import Final, Protocol
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.types.proxy.auth.user_api_key_auth import (
UserAPIKeyAuth,
)
@ -47,15 +48,20 @@ def resolve_resource_owner_id(
return None
def build_list_page(items: list[Any], has_more: bool = False) -> dict[str, object]:
class _Identified(Protocol):
@property
def id(self) -> object: ...
def build_list_page(items: Sequence[object], has_more: bool = False) -> dict[str, object]:
"""Build the OpenAI-style paginated list response shape used by managed
file/batch/vector-store listings. ``first_id`` and ``last_id`` are
sourced from each item's ``.id`` attribute."""
return {
"object": "list",
"data": items,
"first_id": items[0].id if items else None,
"last_id": items[-1].id if items else None,
"first_id": require_attributes(items[0], _Identified, "id").id if items else None,
"last_id": require_attributes(items[-1], _Identified, "id").id if items else None,
"has_more": has_more,
}

15
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -11,7 +11,7 @@ from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.caching.caching import InMemoryCache
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.core_helpers import decoded_json_object, map_finish_reason
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping
from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_request_id_headers
@ -50,6 +50,7 @@ from litellm.types.utils import GenericStreamingChunk as GChunk
from ..common_utils import (
BedrockError,
BedrockEventStreamResponseDict,
bedrock_event_chunk_text,
bedrock_event_stream_header,
bedrock_event_stream_response,
bedrock_stream_event_error_status,
@ -59,6 +60,7 @@ from ..common_utils import (
get_bedrock_response_stream_shape,
get_bedrock_stream_event_statuses,
get_bedrock_tool_name,
parse_bedrock_event_stream,
)
bedrock_tool_name_mappings: Final[InMemoryCache] = InMemoryCache(max_size_in_memory=50, default_ttl=600)
@ -922,7 +924,7 @@ class AWSEventStreamDecoder:
response_stream_shape: "Shape",
request_id_headers: httpx.Headers | None = None,
) -> str | None:
parsed_response: Final = self.parser.parse(response_dict, response_stream_shape)
parsed_response: Final = parse_bedrock_event_stream(self.parser, response_dict, response_stream_shape)
if response_dict["status_code"] != 200:
raise build_bedrock_stream_error(response_dict, response_stream_shape, headers=request_id_headers)
@ -932,11 +934,12 @@ class AWSEventStreamDecoder:
raise build_bedrock_stream_event_error(
event_type, event_error_status, response_dict["body"], headers=request_id_headers
)
if "chunk" in parsed_response:
chunk = parsed_response.get("chunk")
if not chunk:
parsed_fields: Final = decoded_json_object(parsed_response)
if "chunk" in parsed_fields:
event_chunk: Final = parsed_fields.get("chunk")
if not event_chunk:
return None
return chunk.get("bytes").decode()
return bedrock_event_chunk_text(event_chunk)
else:
chunk = response_dict.get("body")
if not chunk:

32
litellm/llms/bedrock/common_utils.py
View file

@ -13,7 +13,7 @@ import warnings
from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias
from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias
from typing_extensions import (
ReadOnly,
@ -24,6 +24,7 @@ from typing_extensions import (
if TYPE_CHECKING:
from botocore.eventstream import EventStreamMessage
from botocore.model import ServiceModel, Shape
from botocore.parsers import EventStreamJSONParser
from litellm.types.llms.bedrock import BedrockCreateBatchRequest
@ -33,7 +34,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError
import litellm
from litellm import verbose_logger
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.core_helpers import lax_bool
from litellm.litellm_core_utils.core_helpers import decoded_json_object, lax_bool, require_attributes
from litellm.litellm_core_utils.optional_imports import ensure_optional_import
from litellm.litellm_core_utils.prompt_templates.json_schema_utils import (
schema_property_count,
@ -135,7 +136,7 @@ _BEDROCK_AWS_AUTH_PARAMETER_KEYS: Final[tuple[str, ...]] = (*AWS_AUTH_PARAM_KEYS
def merge_bedrock_aws_request_params(
litellm_params: Mapping[str, object],
optional_params: Mapping[str, object],
) -> dict[str, Any]:
) -> dict[str, object]:
"""Merge deployment and request parameters without allowing auth escalation.
Deployment configuration is authoritative for AWS authentication. When a
@ -2516,15 +2517,16 @@ class BedrockEventStreamDecoderBase:
),
)
response_dict: Final = event.to_response_dict()
parsed_response: Final = self.parser.parse(response_dict, response_stream_shape)
parsed_response: Final = parse_bedrock_event_stream(self.parser, response_dict, response_stream_shape)
if response_dict["status_code"] != 200:
raise build_bedrock_stream_error(response_dict, response_stream_shape)
if "chunk" in parsed_response:
chunk = parsed_response.get("chunk")
if not chunk:
parsed_fields: Final = decoded_json_object(parsed_response)
if "chunk" in parsed_fields:
event_chunk: Final = parsed_fields.get("chunk")
if not event_chunk:
return None
return chunk.get("bytes").decode()
return bedrock_event_chunk_text(event_chunk)
else:
chunk = response_dict.get("body")
if not chunk:
@ -2541,6 +2543,20 @@ def _decoded_json_value(raw: str) -> object:
return _JSON_VALUE.validate_python(json.loads(raw))
class _DecodablePayload(Protocol):
def decode(self) -> str: ...
def parse_bedrock_event_stream(
parser: EventStreamJSONParser, response_dict: Mapping[str, object], response_stream_shape: Shape
) -> object:
return _JSON_VALUE.validate_python(parser.parse(response_dict, response_stream_shape))
def bedrock_event_chunk_text(chunk: object) -> str:
return require_attributes(decoded_json_object(chunk).get("bytes"), _DecodablePayload, "decode").decode()
def get_anthropic_beta_from_headers(headers: dict) -> list[str]:
"""
Extract anthropic-beta header values and convert them to a list.

2
litellm/llms/bedrock/count_tokens/bedrock_token_counter.py
View file

@ -45,7 +45,7 @@ class _CountTokensReply(LiteLLMPydanticObjectBase):
@dataclass(frozen=True, slots=True)
class CountedTokens:
input_tokens: int
original_response: Mapping[str, JsonValue]
original_response: Mapping[str, object]
tokenizer_type: str

10
litellm/llms/bedrock/count_tokens/handler.py
View file

@ -7,14 +7,18 @@ Simplified handler leveraging existing LiteLLM Bedrock infrastructure.
from typing import Any, Final
import httpx
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import decoded_json_object
from litellm.llms.bedrock.base_aws_llm import run_aws_signing
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
_DECODED_JSON: Final = TypeAdapter(object)
class BedrockCountTokensHandler(BedrockCountTokensConfig):
"""
@ -29,7 +33,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
litellm_params: dict[str, Any],
resolved_model: str,
client: AsyncHTTPHandler | None = None,
) -> dict[str, Any]:
) -> dict[str, object]:
"""
Handle a CountTokens request using existing LiteLLM patterns.
@ -111,12 +115,12 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
response=response,
)
bedrock_response: Final = response.json()
bedrock_response: Final = _DECODED_JSON.validate_python(response.json())
verbose_logger.debug("Bedrock response: %s", bedrock_response)
# Transform response back to expected format
final_response: Final = self.transform_bedrock_response_to_anthropic(bedrock_response)
final_response: Final = self.transform_bedrock_response_to_anthropic(decoded_json_object(bedrock_response))
verbose_logger.debug("Final response: %s", final_response)

4
litellm/llms/bedrock/count_tokens/transformation.py
View file

@ -247,9 +247,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
)
return join_api_base(base_url, f"model/{encoded_model_id}/count-tokens")
def transform_bedrock_response_to_anthropic(
self, bedrock_response: Mapping[str, JsonValue]
) -> dict[str, JsonValue]:
def transform_bedrock_response_to_anthropic(self, bedrock_response: Mapping[str, object]) -> dict[str, object]:
"""
Transform Bedrock CountTokens response to Anthropic format.

25
litellm/llms/bedrock/files/transformation.py
View file

@ -11,7 +11,7 @@ from datetime import datetime
from email.utils import parsedate_to_datetime
from itertools import chain
from types import MappingProxyType
from typing import Any, Final, Literal, TypeAlias, TypedDict
from typing import Any, Final, Literal, Protocol, TypeAlias, TypedDict
from urllib.parse import quote, unquote, urlencode
import httpx
@ -19,6 +19,7 @@ from httpx import Headers, Response
from openai.types.file_deleted import FileDeleted
from pydantic import ConfigDict, Field, TypeAdapter
from typing_extensions import ReadOnly
from typing_extensions import TypedDict as ExtraItemsTypedDict
from litellm._logging import verbose_logger
from litellm._uuid import uuid
@ -38,6 +39,7 @@ from litellm.litellm_core_utils.cloud_storage_security import (
split_configured_cloud_bucket_name,
validate_managed_cloud_file_id,
)
from litellm.litellm_core_utils.core_helpers import require_attributes
from litellm.litellm_core_utils.prompt_templates.common_utils import (
text_completion_prompt_to_messages,
)
@ -151,6 +153,10 @@ def _retrieved_s3_file_size(raw_response: Response) -> int:
)
class _SupportsRstrip(Protocol):
def rstrip(self, chars: str, /) -> str: ...
class _S3DeleteContext(LiteLLMBaseModel):
file_id: str = Field(min_length=1)
@ -168,6 +174,10 @@ _LITELLM_PARAMS_THE_MAPPER_TAKES: Final = frozenset({"allowed_openai_params"})
_MAPPED_PARAMS_THE_REQUEST_HANDLER_STRIPS: Final = frozenset({"json_mode"})
class _ParamsTheMapperOwns(ExtraItemsTypedDict, total=False, extra_items=object):
pass
_UPLOAD_PARAMS: Final = TypeAdapter(dict[str, object])
@ -710,9 +720,11 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
encoded_object_name: Final = encode_s3_object_key_for_url(object_name)
# S3 endpoint URL format
s3_endpoint_url: Final = (
s3_endpoint_url: Final = require_attributes(
request_params.get("s3_endpoint_url")
or f"https://s3.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}"
or f"https://s3.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}",
_SupportsRstrip,
"rstrip",
).rstrip("/")
return f"{s3_endpoint_url}/{bucket_name}/{encoded_object_name}"
@ -1062,7 +1074,8 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
from litellm.types.utils import LlmProviders
messages: Final = map_developer_role_to_system_role(openai_request_body.get("messages", []))
optional_params: Final = {k: v for k, v in openai_request_body.items() if k not in ["model", "messages"]}
request_params: Final[Mapping[str, object]] = openai_request_body
optional_params: Final = {k: v for k, v in request_params.items() if k not in ["model", "messages"]}
# --- Anthropic: use existing AmazonAnthropicClaudeConfig ---
if provider == LlmProviders.ANTHROPIC:
@ -1085,8 +1098,8 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
custom_llm_provider="bedrock",
messages=messages,
model_info=deployment_model_info(litellm_params),
**MappingProxyType(
{
**_ParamsTheMapperOwns(
**{
k: v
for k, v in optional_params.items()
if not is_litellm_owned_kwarg(k) or k in _LITELLM_PARAMS_THE_MAPPER_TAKES

17
litellm/llms/bedrock/image_edit/amazon_nova_canvas_image_edit_transformation.py
View file

@ -18,8 +18,10 @@ from typing import TYPE_CHECKING, Any, Final, cast
import httpx
from httpx._types import RequestFiles
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import decoded_json_object, iter_items
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.types.images.main import ImageEditOptionalRequestParams
@ -38,6 +40,8 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
_DECODED_JSON: Final = TypeAdapter(object)
def _nova_canvas_task_body(
*,
@ -401,7 +405,7 @@ class BedrockAmazonNovaCanvasImageEditConfig(BaseImageEditConfig):
json_mode: bool | None = None,
) -> ImageResponse:
try:
response_data: Final = raw_response.json()
response_data: Final = _DECODED_JSON.validate_python(raw_response.json())
except Exception as e:
raise self.get_error_class(
error_message=f"Error parsing Nova Canvas image edit response: {e}",
@ -416,18 +420,19 @@ class BedrockAmazonNovaCanvasImageEditConfig(BaseImageEditConfig):
headers=raw_response.headers,
)
images: Final[list[str]] = response_data.get("images") or []
response_object: Final = decoded_json_object(response_data)
images: Final[object] = response_object.get("images") or []
if "errors" in response_data and not images:
if "errors" in response_object and not images:
raise self.get_error_class(
error_message=f"Nova Canvas image edit error: {response_data['errors']}",
error_message=f"Nova Canvas image edit error: {response_object['errors']}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
# Nova Canvas InvokeModel success body uses "images" and optional "error" (AWS docs);
# it does not use Stability-style "finish_reasons".
error_msg = response_data.get("message") or response_data.get("error")
error_msg = response_object.get("message") or response_object.get("error")
if error_msg and not images:
if not isinstance(error_msg, str):
error_msg = str(error_msg)
@ -439,7 +444,7 @@ class BedrockAmazonNovaCanvasImageEditConfig(BaseImageEditConfig):
model_response: Final = ImageResponse()
model_response.data = []
for image_b64 in images:
for image_b64 in iter_items(images):
if image_b64:
model_response.data.append(
ImageObject(

34
litellm/llms/bedrock/image_edit/stability_transformation.py
View file

@ -22,11 +22,13 @@ API Reference: https://docs.aws.amazon.com/bedrock/latest/userguide/model-parame
"""
import base64
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final, Protocol, cast, runtime_checkable
import httpx
from httpx._types import RequestFiles
from pydantic import TypeAdapter
from litellm.litellm_core_utils.core_helpers import decoded_json_object, iter_items, json_contains
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.llms.bedrock.common_utils import BedrockError
from litellm.types.images.main import ImageEditOptionalRequestParams
@ -44,6 +46,19 @@ if TYPE_CHECKING:
else:
LiteLLMLoggingObj = Any
_DECODED_JSON: Final = TypeAdapter(object)
@runtime_checkable
class _SupportsGetItem(Protocol):
def __getitem__(self, key: str | int, /) -> object: ...
def _subscript(value: object, key: str | int) -> object:
if isinstance(value, _SupportsGetItem):
return value[key]
raise TypeError(f"'{type(value).__name__}' object is not subscriptable")
class BedrockStabilityImageEditConfig(BaseImageEditConfig):
"""
@ -299,7 +314,7 @@ class BedrockStabilityImageEditConfig(BaseImageEditConfig):
OpenAI expects: {"data": [{"b64_json": "base64..."}], "created": timestamp}
"""
try:
response_data: Final = raw_response.json()
response_data: Final = _DECODED_JSON.validate_python(raw_response.json())
except Exception as e:
raise self.get_error_class(
error_message=f"Error parsing Bedrock Stability response: {e}",
@ -308,18 +323,19 @@ class BedrockStabilityImageEditConfig(BaseImageEditConfig):
)
# Check for errors in response
if "errors" in response_data:
if json_contains(response_data, "errors"):
raise self.get_error_class(
error_message=f"Bedrock Stability error: {response_data['errors']}",
error_message=f"Bedrock Stability error: {_subscript(response_data, 'errors')}",
status_code=raw_response.status_code,
headers=raw_response.headers,
)
# Check finish_reasons
finish_reasons: Final = response_data.get("finish_reasons", [])
if finish_reasons and finish_reasons[0]:
response_fields: Final = decoded_json_object(response_data)
finish_reasons: Final = response_fields.get("finish_reasons", [])
if finish_reasons and _subscript(finish_reasons, 0):
raise self.get_error_class(
error_message=f"Bedrock Stability error: {finish_reasons[0]}",
error_message=f"Bedrock Stability error: {_subscript(finish_reasons, 0)}",
status_code=400,
headers=raw_response.headers,
)
@ -329,9 +345,9 @@ class BedrockStabilityImageEditConfig(BaseImageEditConfig):
model_response.data = []
# Extract images from response
images: Final = response_data.get("images", [])
images: Final = response_fields.get("images", [])
if images:
for image_b64 in images:
for image_b64 in iter_items(images):
if image_b64:
model_response.data.append(
ImageObject(

5
litellm/llms/bedrock/passthrough/guardrail_translation/handler.py
View file

@ -14,7 +14,7 @@ from pydantic import InstanceOf, TypeAdapter, ValidationError
from typing_extensions import ReadOnly, TypedDict
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.core_helpers import as_str_mapping
from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object
from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler
from litellm.llms.base_llm.guardrail_translation.base_translation import (
BaseTranslation,
@ -849,6 +849,7 @@ class InvokeStreamFrame:
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
_DECODED_JSON: Final = TypeAdapter(object)
_Frame: Final = TypeVar("_Frame", ConverseStreamFrame, InvokeStreamFrame)
@ -1298,7 +1299,7 @@ class BedrockPassthroughGuardrailHandler(BaseTranslation):
texts: Sequence[tuple[_StreamGroupKey, str]] = ()
if event_type == "contentBlockDelta":
try:
payload_dict: dict[str, object] = json.loads(payload_bytes)
payload_dict = decoded_json_object(_DECODED_JSON.validate_python(json.loads(payload_bytes)))
texts = _delta_frame_texts(payload_dict)
except Exception as e:
verbose_proxy_logger.debug(

266
litellm/llms/bedrock/realtime/handler.py
View file

@ -6,12 +6,14 @@ This uses aws_sdk_bedrock_runtime for bidirectional streaming with Nova Sonic.
import asyncio
import contextlib
import functools
import importlib.metadata
import json
from collections.abc import AsyncIterator, Awaitable, Callable, Mapping, MutableMapping, Sequence
from collections.abc import AsyncIterator, Awaitable, Callable, Iterable, Mapping, MutableMapping, Sequence
from dataclasses import dataclass, field
from enum import Enum, auto
from types import MappingProxyType
from typing import Final, NoReturn, Protocol, runtime_checkable
from typing import TYPE_CHECKING, Final, NoReturn, Protocol, runtime_checkable
from pydantic import JsonValue, TypeAdapter
@ -24,11 +26,25 @@ from litellm.constants import (
BEDROCK_REALTIME_SDK_DISTRIBUTION,
BEDROCK_REALTIME_SDK_SUPPORTED_RANGE,
BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY,
REALTIME_SESSION_ENDED_ABNORMALLY_KEY,
REALTIME_SESSION_SUCCESS_LOGGED_KEY,
)
from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.litellm_core_utils.realtime_errors import realtime_error_event
from litellm.litellm_core_utils.realtime_guardrails import (
RealtimeGuardedItem,
RealtimeGuardrailBlock,
RealtimeGuardrailRewrite,
RealtimeGuardrailVerdict,
RealtimeToolOutput,
guardrail_violation_error_event,
guardrail_voice_prompt_events,
realtime_guarded_item,
realtime_guardrail_verdict,
rewritten_item,
)
from litellm.litellm_core_utils.realtime_streaming import DefaultLoggedRealTimeEventTypes
from litellm.llms.base_llm.realtime.transformation import RealtimeSessionUpdateRejection
from litellm.types.llms.bedrock import AwsAuthParams
@ -40,9 +56,13 @@ from ..base_aws_llm import BaseAWSLLM, run_aws_signing
from ..common_utils import BedrockError
from .transformation import BedrockRealtimeConfig, requested_modalities
if TYPE_CHECKING:
from litellm.types.guardrails import GuardrailEventHooks
_CLIENT_MESSAGE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
_REQUEST_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
_SPEND_LOG_METADATA_KEYS: Final = ("litellm_metadata", "metadata")
GUARDRAIL_UNAVAILABLE_MESSAGE: Final = "A guardrail could not check this request, so the session was ended"
_EMPTY_JSON_OBJECT: Final[Mapping[str, JsonValue]] = MappingProxyType({})
@ -259,11 +279,11 @@ def _raise_provider_failure(scope: MutableMapping[str, object], failure: BaseExc
raise error from failure
def _parse_client_message(message: str) -> Mapping[str, JsonValue]:
def _parse_client_message(message: str) -> Mapping[str, JsonValue] | None:
try:
return _json_dict(_CLIENT_MESSAGE_ADAPTER.validate_json(message))
except ValueError:
return _EMPTY_JSON_OBJECT
return None
async def _send_opening_message(
@ -282,6 +302,27 @@ async def _send_opening_message(
client_ws.scope[BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY] = True # rebind-ok: scope outlives the attempt
async def _forward_client_message(
client_ws: RealtimeClientWebSocket,
bedrock_stream: BedrockBidirectionalStream,
transformation_config: BedrockRealtimeConfig,
model: str,
session_state: RealtimeResponseTransformInput,
send_to_bedrock: Callable[[str], Awaitable[None]],
client_message: str,
) -> None:
transformed_messages: Final = transformation_config.transform_realtime_request(
message=client_message,
model=model,
session_configuration_request=session_state.get("session_configuration_request"),
)
if transformed_messages and not _is_session_committed(client_ws.scope):
await _send_opening_message(client_ws, bedrock_stream, client_message, transformed_messages, send_to_bedrock)
return
for bedrock_message in transformed_messages:
await send_to_bedrock(bedrock_message)
async def _ack_session_update(
client_ws: RealtimeClientWebSocket,
transformation_config: BedrockRealtimeConfig,
@ -317,6 +358,161 @@ async def _reject_session_update(
return True
async def _send_guardrail_violation(client_ws: RealtimeClientWebSocket, block: RealtimeGuardrailBlock) -> bool:
try:
await client_ws.send_text(guardrail_violation_error_event(block.message))
except Exception as e: # noqa: BLE001 # any send failure means the client is gone
verbose_proxy_logger.debug("Bedrock Realtime: guardrail error not delivered: %s", e, exc_info=True)
return False
return True
async def _end_guarded_session(client_ws: RealtimeClientWebSocket) -> None:
verbose_proxy_logger.warning("Bedrock Realtime: ending session after a guardrail violation")
with contextlib.suppress(Exception):
await client_ws.close()
async def _end_unscreened_session(client_ws: RealtimeClientWebSocket) -> None:
with contextlib.suppress(Exception):
await client_ws.send_text(realtime_error_event(GUARDRAIL_UNAVAILABLE_MESSAGE, error_type="guardrail_error"))
with contextlib.suppress(Exception):
await client_ws.close()
@dataclass(frozen=True, slots=True)
class _GuardrailUnavailable:
pass
def _registered_callbacks() -> Iterable[object]:
return litellm.callbacks
class _ClientEventRoute(Enum):
FORWARD = auto()
DROP = auto()
END_SESSION = auto()
class BedrockRealtimeGuardrails:
"""One Bedrock realtime session's guardrail checks, shared by both bridge directions.
Nova Sonic has no way to cancel a response or remove a turn, so a blocked user transcript ends
the session before that turn's answer reaches the client.
"""
def __init__(
self,
request_data: Mapping[str, object] = _EMPTY_JSON_OBJECT,
user_api_key_dict: object | None = None,
callbacks: Callable[[], Iterable[object]] = _registered_callbacks,
) -> None:
self._request_data: Final = request_data
self._user_api_key_dict: Final = user_api_key_dict
self._callbacks: Final = callbacks
self._violation_count = 0
self._drop_next_response_create = False
self._failed = False
@property
def failed(self) -> bool:
return self._failed
async def _verdict(
self, texts: tuple[str, ...], role: str, hook: "GuardrailEventHooks"
) -> RealtimeGuardrailVerdict | _GuardrailUnavailable:
try:
return await realtime_guardrail_verdict(
texts, role, (hook,), self._request_data, self._user_api_key_dict, self._callbacks()
)
except Exception as e: # noqa: BLE001 # a guardrail that cannot decide fails closed, never as a Bedrock failure
verbose_proxy_logger.warning("Bedrock Realtime: a guardrail failed, ending the session: %s", e)
self._failed = True
return _GuardrailUnavailable()
def _ends_session(self, block: RealtimeGuardrailBlock) -> bool:
self._violation_count += 1
return block.ends_session(self._violation_count)
async def screen_client_event(
self,
client_event: Mapping[str, JsonValue],
client_ws: RealtimeClientWebSocket,
forward_to_bedrock: Callable[[str], Awaitable[None]],
) -> _ClientEventRoute:
from litellm.types.guardrails import GuardrailEventHooks
event_type: Final = client_event.get("type")
if event_type == "response.create" and self._drop_next_response_create:
self._drop_next_response_create = False
return _ClientEventRoute.DROP
if event_type == "conversation.item.create":
self._drop_next_response_create = False
guarded_item: Final = realtime_guarded_item(client_event)
if guarded_item is None:
return _ClientEventRoute.FORWARD
held: Final = json.dumps(client_event)
client_ws.scope[BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY] = held # rebind-ok: replayed on fallback
role: Final = "tool" if isinstance(guarded_item, RealtimeToolOutput) else "user"
match await self._verdict(guarded_item.texts, role, GuardrailEventHooks.pre_call):
case None:
return _ClientEventRoute.FORWARD
case RealtimeGuardrailRewrite(texts=texts):
await forward_to_bedrock(json.dumps({**client_event, "item": rewritten_item(client_event, texts)}))
return _ClientEventRoute.DROP
case RealtimeGuardrailBlock() as block:
return await self._route_blocked_item(block, guarded_item, client_ws, forward_to_bedrock)
case _GuardrailUnavailable():
await _end_unscreened_session(client_ws)
return _ClientEventRoute.END_SESSION
async def _route_blocked_item(
self,
block: RealtimeGuardrailBlock,
guarded_item: RealtimeGuardedItem,
client_ws: RealtimeClientWebSocket,
forward_to_bedrock: Callable[[str], Awaitable[None]],
) -> _ClientEventRoute:
if isinstance(guarded_item, RealtimeToolOutput):
await forward_to_bedrock(guarded_item.sanitized_event)
if not await _send_guardrail_violation(client_ws, block):
return _ClientEventRoute.END_SESSION
if self._ends_session(block):
await _end_guarded_session(client_ws)
return _ClientEventRoute.END_SESSION
if isinstance(guarded_item, RealtimeToolOutput):
return _ClientEventRoute.DROP
for voice_prompt_event in guardrail_voice_prompt_events(f'"{block.message}"'):
await forward_to_bedrock(voice_prompt_event)
self._drop_next_response_create = True
return _ClientEventRoute.DROP
async def continues_after(
self, provider_event: Mapping[str, object], client_ws: RealtimeClientWebSocket, client_audio_streamed: bool
) -> bool:
from litellm.types.guardrails import GuardrailEventHooks
transcript: Final = provider_event.get("transcript")
if (
not client_audio_streamed
or provider_event.get("type") != "conversation.item.input_audio_transcription.completed"
or not isinstance(transcript, str)
or not transcript
):
return True
match await self._verdict((transcript,), "user", GuardrailEventHooks.realtime_input_transcription):
case RealtimeGuardrailBlock() as block:
if await _send_guardrail_violation(client_ws, block):
await _end_guarded_session(client_ws)
return False
case _GuardrailUnavailable():
await _end_unscreened_session(client_ws)
return False
case None | RealtimeGuardrailRewrite():
return True
async def _close_bedrock_input(
bedrock_stream: BedrockBidirectionalStream,
close_messages: Sequence[str],
@ -381,6 +577,8 @@ class BedrockRealtime(BaseAWSLLM):
aws_external_id: str | None = None,
aws_session_tags: object = None,
drop_params: bool = False,
user_api_key_dict: object | None = None,
litellm_metadata: Mapping[str, object] | None = None,
**kwargs: object,
):
"""
@ -461,7 +659,15 @@ class BedrockRealtime(BaseAWSLLM):
try:
await self._run_session(
websocket, open_bidirectional_stream, model, logging_obj, pending_opening_message, drop_params
websocket,
open_bidirectional_stream,
model,
logging_obj,
pending_opening_message,
drop_params,
BedrockRealtimeGuardrails(
request_data={"litellm_metadata": litellm_metadata or {}}, user_api_key_dict=user_api_key_dict
),
)
finally:
await _close_bedrock_client(bedrock_client)
@ -474,6 +680,7 @@ class BedrockRealtime(BaseAWSLLM):
logging_obj: LiteLLMLogging,
pending_opening_message: str | None,
drop_params: bool = False,
guardrails: BedrockRealtimeGuardrails | None = None,
) -> None:
transformation_config: Final = BedrockRealtimeConfig(drop_params=drop_params)
@ -496,6 +703,7 @@ class BedrockRealtime(BaseAWSLLM):
"session_configuration_request": None,
}
session_guardrails: Final = guardrails if guardrails is not None else BedrockRealtimeGuardrails()
outcome: Final = await self._bridge(
websocket,
bedrock_stream,
@ -504,7 +712,10 @@ class BedrockRealtime(BaseAWSLLM):
session_state,
logging_obj,
initial_message=pending_opening_message,
guardrails=session_guardrails,
)
if session_guardrails.failed:
logging_obj.model_call_details[REALTIME_SESSION_ENDED_ABNORMALLY_KEY] = True
logged_events: Final = (
*outcome.logged_events,
@ -544,10 +755,12 @@ class BedrockRealtime(BaseAWSLLM):
session_state: RealtimeResponseTransformInput,
logging_obj: LiteLLMLogging,
initial_message: str | None,
guardrails: BedrockRealtimeGuardrails | None = None,
) -> _BridgeOutcome:
"""Run both forwarding directions until the client leaves or either side fails."""
logged: Final[list[OpenAIRealtimeEvents]] = [] # mutable-ok: events forwarded before a failure are still spend
signals: Final = SessionEndSignals()
session_guardrails: Final = guardrails if guardrails is not None else BedrockRealtimeGuardrails()
async def collect_logged_events() -> None:
try:
@ -559,6 +772,7 @@ class BedrockRealtime(BaseAWSLLM):
logging_obj,
session_state,
signals=signals,
guardrails=session_guardrails,
):
logged.append(event)
except Exception:
@ -575,6 +789,7 @@ class BedrockRealtime(BaseAWSLLM):
logging_obj,
initial_message,
signals=signals,
guardrails=session_guardrails,
)
)
bedrock_task: Final = asyncio.create_task(collect_logged_events())
@ -610,6 +825,7 @@ class BedrockRealtime(BaseAWSLLM):
initial_message: str | None = None,
*,
signals: SessionEndSignals,
guardrails: BedrockRealtimeGuardrails | None = None,
) -> None:
"""Forward messages from client WebSocket to Bedrock stream.
@ -631,10 +847,23 @@ class BedrockRealtime(BaseAWSLLM):
raise _stream_ended_before_session_closed() from e
verbose_proxy_logger.debug("Bedrock Realtime: Sent to Bedrock: %s", bedrock_message[:200])
forward_to_bedrock: Final = functools.partial(
_forward_client_message,
client_ws,
bedrock_stream,
transformation_config,
model,
session_state,
send_to_bedrock,
)
session_guardrails: Final = guardrails if guardrails is not None else BedrockRealtimeGuardrails()
try:
async for message in _client_messages(client_ws, initial_message):
verbose_proxy_logger.debug("Bedrock Realtime: Received from client: %s", message[:200])
parsed_client_message = _parse_client_message(message)
if parsed_client_message is None:
verbose_proxy_logger.warning("Bedrock Realtime: dropped a client message that is not valid JSON")
continue
is_session_update = _json_str(parsed_client_message.get("type")) == "session.update"
if is_session_update:
rejection = transformation_config.session_update_rejection(parsed_client_message)
@ -643,18 +872,15 @@ class BedrockRealtime(BaseAWSLLM):
break
continue
transformed_messages = transformation_config.transform_realtime_request(
message=message,
model=model,
session_configuration_request=session_state.get("session_configuration_request"),
)
if transformed_messages and not _is_session_committed(client_ws.scope):
await _send_opening_message(
client_ws, bedrock_stream, message, transformed_messages, send_to_bedrock
)
else:
for bedrock_message in transformed_messages:
await send_to_bedrock(bedrock_message)
match await session_guardrails.screen_client_event(
parsed_client_message, client_ws, forward_to_bedrock
):
case _ClientEventRoute.END_SESSION:
break
case _ClientEventRoute.DROP:
continue
case _ClientEventRoute.FORWARD:
await forward_to_bedrock(message)
if is_session_update and not await _ack_session_update(
client_ws, transformation_config, model, logging_obj, parsed_client_message
@ -679,6 +905,7 @@ class BedrockRealtime(BaseAWSLLM):
session_state: RealtimeResponseTransformInput,
*,
signals: SessionEndSignals,
guardrails: BedrockRealtimeGuardrails | None = None,
) -> AsyncIterator[OpenAIRealtimeEvents]:
"""Forward messages from Bedrock to the client, yielding the ones to record for spend logging.
@ -694,6 +921,7 @@ class BedrockRealtime(BaseAWSLLM):
verbose_proxy_logger.debug("Bedrock Realtime: Sent to client: %s", message_json[:200])
return True
session_guardrails: Final = guardrails if guardrails is not None else BedrockRealtimeGuardrails()
output: Final = await bedrock_stream.await_output()
while True:
result = await output[1].receive()
@ -750,3 +978,7 @@ class BedrockRealtime(BaseAWSLLM):
return
if _should_log_event(openai_message):
yield openai_message
if not await session_guardrails.continues_after(
openai_message, client_ws, transformation_config.client_audio_streamed
):
return

54
litellm/llms/bedrock/vector_stores/transformation.py
View file

@ -1,11 +1,12 @@
from collections.abc import Mapping
from copy import deepcopy
from typing import TYPE_CHECKING, Any, Final, cast
from typing import TYPE_CHECKING, Any, Final, Protocol, cast, runtime_checkable
from urllib.parse import urlparse
import httpx
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import decoded_json_object
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
@ -34,6 +35,32 @@ else:
LiteLLMLoggingObj = Any
@runtime_checkable
class _SupportsSetDefault(Protocol):
def setdefault(self, key: str, default: object, /) -> object: ...
@runtime_checkable
class _SupportsSetItem(Protocol):
def __setitem__(self, key: str, value: object, /) -> None: ...
def _setdefault_target(value: object) -> _SupportsSetDefault:
if isinstance(value, _SupportsSetDefault):
return value
raise AttributeError(f"{type(value).__name__!r} object has no attribute 'setdefault'", name="setdefault", obj=value)
def _item_assignment_target(value: object) -> _SupportsSetItem:
if isinstance(value, _SupportsSetItem):
return value
raise TypeError(f"'{type(value).__name__}' object does not support item assignment")
def _vector_search_configuration(retrieval_config: object) -> Mapping[str, object]:
return decoded_json_object(decoded_json_object(retrieval_config).get("vectorSearchConfiguration", {}))
class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
"""Vector store configuration for AWS Bedrock Knowledge Bases."""
@ -206,7 +233,7 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
api_base: str,
litellm_logging_obj: LiteLLMLoggingObj,
litellm_params: dict,
extra_body: dict[str, Any] | None = None,
extra_body: dict[str, object] | None = None,
) -> tuple[str, dict]:
if isinstance(query, list):
query = " ".join(query)
@ -218,30 +245,33 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
"retrievalQuery": BedrockKBRetrievalQuery(text=query),
}
retrieval_config: dict[str, Any] = {}
if isinstance(extra_body, dict):
retrieval_config = deepcopy(
extra_body.get("retrievalConfiguration") or extra_body.get("retrieval_configuration") or {}
)
retrieval_config: Final[object] = (
deepcopy(extra_body.get("retrievalConfiguration") or extra_body.get("retrieval_configuration") or {})
if isinstance(extra_body, dict)
else {}
)
max_results: Final = vector_store_search_optional_params.get("max_num_results")
if max_results is not None:
existing_number_of_results = retrieval_config.get("vectorSearchConfiguration", {}).get("numberOfResults")
existing_number_of_results: Final = _vector_search_configuration(retrieval_config).get("numberOfResults")
if existing_number_of_results is not None and existing_number_of_results != max_results:
verbose_logger.debug(
"Overriding extra_body retrievalConfiguration.vectorSearchConfiguration.numberOfResults (%s) with max_num_results=%s",
existing_number_of_results,
max_results,
)
retrieval_config.setdefault("vectorSearchConfiguration", {})["numberOfResults"] = max_results
_item_assignment_target(_setdefault_target(retrieval_config).setdefault("vectorSearchConfiguration", {}))[
"numberOfResults"
] = max_results
filters: Final = vector_store_search_optional_params.get("filters")
if filters is not None:
existing_filter: Final = retrieval_config.get("vectorSearchConfiguration", {}).get("filter")
existing_filter: Final = _vector_search_configuration(retrieval_config).get("filter")
if existing_filter is not None and existing_filter != filters:
verbose_logger.debug(
"Overriding extra_body retrievalConfiguration.vectorSearchConfiguration.filter with filters from vector_store_search_optional_params"
)
retrieval_config.setdefault("vectorSearchConfiguration", {})["filter"] = filters
_item_assignment_target(_setdefault_target(retrieval_config).setdefault("vectorSearchConfiguration", {}))[
"filter"
] = filters
if retrieval_config:
request_body["retrievalConfiguration"] = cast(BedrockKBRetrievalConfiguration, retrieval_config)
user_context: Final = self._user_context(extra_body=extra_body, litellm_params=litellm_params)

34
litellm/llms/black_forest_labs/image_edit/transformation.py
View file

@ -10,7 +10,7 @@ API Reference: https://docs.bfl.ai/
import base64
import time
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final
from typing import TYPE_CHECKING, Any, Final, Protocol
import httpx
from httpx._types import RequestFiles
@ -19,6 +19,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
import litellm
from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH
from litellm.litellm_core_utils.core_helpers import as_object_list, has_attributes
from litellm.litellm_core_utils.url_utils import async_safe_get, join_api_base, safe_get
from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
from litellm.secret_managers.main import get_secret_str
@ -52,6 +53,19 @@ class _BflPolledResult(TypedDict):
_BFL_POLLED_RESULT_ADAPTER: Final = TypeAdapter(_BflPolledResult)
_DECODED_JSON: Final = TypeAdapter(object)
_UNTYPED_VALUE: Final = TypeAdapter(object)
class _BinaryReadable(Protocol):
def read(self) -> bytes: ...
class _Tellable(Protocol):
def tell(self) -> int: ...
class _Seekable(Protocol):
def seek(self, offset: int, /) -> object: ...
_BFL_REQUEST_PARAMS: Final = (
@ -197,7 +211,7 @@ class BlackForestLabsImageEditConfig(BaseImageEditConfig):
def _read_image_bytes(
self,
image: Any,
image: object,
depth: int = 0,
max_depth: int = DEFAULT_MAX_RECURSE_DEPTH,
) -> bytes:
@ -208,9 +222,8 @@ class BlackForestLabsImageEditConfig(BaseImageEditConfig):
)
if isinstance(image, bytes):
return image
elif isinstance(image, list):
# If it's a list, take the first image
return self._read_image_bytes(image[0], depth=depth + 1, max_depth=max_depth)
elif (image_list := as_object_list(image)) is not None:
return self._read_image_bytes(image_list[0], depth=depth + 1, max_depth=max_depth)
elif isinstance(image, str):
if image.startswith(("http://", "https://")):
response: Final = safe_get(litellm.module_level_client, image, timeout=60.0)
@ -221,13 +234,12 @@ class BlackForestLabsImageEditConfig(BaseImageEditConfig):
"Unsupported image input: plain string values that are not URLs are not accepted. "
"Provide image bytes or a file-like object."
)
elif hasattr(image, "read"):
# File-like object
pos: Final = getattr(image, "tell", lambda: 0)()
if hasattr(image, "seek"):
elif has_attributes(image, _BinaryReadable, "read"):
pos: Final = image.tell() if has_attributes(image, _Tellable, "tell") else 0
if has_attributes(image, _Seekable, "seek"):
image.seek(0)
data: Final = image.read()
if hasattr(image, "seek"):
if has_attributes(image, _Seekable, "seek"):
image.seek(pos)
return data
else:
@ -265,7 +277,7 @@ class BlackForestLabsImageEditConfig(BaseImageEditConfig):
# Handle mask if provided (for inpainting)
if "mask" in image_edit_optional_request_params:
mask: Final = image_edit_optional_request_params["mask"]
mask: Final = _UNTYPED_VALUE.validate_python(image_edit_optional_request_params["mask"])
mask_bytes: Final = self._read_image_bytes(mask)
request_body["mask"] = base64.b64encode(mask_bytes).decode("utf-8")

8
litellm/llms/bytez/chat/transformation.py
View file

@ -6,7 +6,9 @@ from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final
import httpx
from pydantic import TypeAdapter
from litellm.litellm_core_utils.core_helpers import decoded_json_object
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.litellm_core_utils.url_utils import encode_url_path_segments
@ -36,6 +38,8 @@ else:
# 5 minute timeout (models may need to load)
STREAMING_TIMEOUT: Final = 60 * 5
_DECODED_JSON: Final = TypeAdapter(object)
class BytezChatConfig(BaseConfig):
"""
@ -194,11 +198,11 @@ class BytezChatConfig(BaseConfig):
) -> ModelResponse:
json: Final = raw_response.json()
error: Final = json.get("error")
error: Final = decoded_json_object(_DECODED_JSON.validate_python(json)).get("error")
if error is not None:
raise BytezError(
message=str(json["error"]),
message=str(error),
status_code=raw_response.status_code,
)

10
litellm/llms/cohere/common_utils.py
View file

@ -1,6 +1,8 @@
import json
from typing import Final, Literal
from pydantic import TypeAdapter
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.cohere import CohereV2StreamedToolCall
@ -12,6 +14,8 @@ from litellm.types.utils import (
ProviderSpecificModelInfo,
)
_DECODED_JSON: Final = TypeAdapter(object)
class CohereError(BaseLLMException):
def __init__(self, status_code, message):
@ -190,8 +194,10 @@ class ModelResponseIterator:
if index != -1:
str_line = str_line[index:]
data_json: Final = json.loads(str_line)
return self.chunk_parser(chunk=data_json)
data_json: Final = _DECODED_JSON.validate_python(json.loads(str_line))
if isinstance(data_json, dict):
return self.chunk_parser(chunk=data_json)
raise AttributeError(f"{type(data_json).__name__!r} object has no attribute 'get'", name="get", obj=data_json)
# Async iterator
def __aiter__(self):

38
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -153,7 +153,7 @@ from litellm.types.containers.main import (
)
from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse
from litellm.types.extraction import ExtractionResponse
from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig
from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig, TwoStepFileUploadRequest
from litellm.types.integrations.custom_logger import (
NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
AgenticLoopLimitError,
@ -188,6 +188,7 @@ from litellm.types.responses.main import DeleteResponseResult
from litellm.types.responses.streaming_websocket import ResponsesWebSocketRequestDefaults
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import (
AgenticLoopParams,
CallTypes,
EmbeddingResponse,
FileTypes,
@ -239,6 +240,10 @@ def _client_request_headers(
return _merge_headers_case_insensitive(optional_headers or {}, extra_headers)
def _requested_stream(response_api_optional_request_params: Mapping[str, object]) -> object:
return response_api_optional_request_params.get("stream", False)
def _rust_responses_websocket_enabled(
custom_llm_provider: str | None,
) -> bool:
@ -386,12 +391,19 @@ class _SignedBodyKwargs(TypedDict, total=False):
json: ReadOnly[dict[str, object]]
class _AgenticLoopParamsView(TypedDict):
agentic_loop_params: ReadOnly[AgenticLoopParams]
_GOAWAY_RETRY_MESSAGE: Final = "Vector store search connection was closed by an HTTP/2 GOAWAY, retrying once"
def _closed_by_http2_goaway(error: httpx.RemoteProtocolError) -> bool:
cause: Final = error.__cause__
return cause is not None and any(isinstance(arg, ConnectionTerminated) for arg in cause.args)
if cause is None:
return False
cause_args: Final[tuple[object, ...]] = cause.args
return any(isinstance(arg, ConnectionTerminated) for arg in cause_args)
async def _asend_with_goaway_retry(send: Callable[[], Awaitable[httpx.Response]]) -> httpx.Response:
@ -433,8 +445,12 @@ def _signed_body_kwargs(*, signed_body: bytes | None, data: dict[str, object]) -
return {"json": data}
def _response_content_type(headers: Mapping[str, str]) -> str:
return headers.get("content-type", "")
def _answered_json(response: httpx.Response) -> bool:
return "application/json" in response.headers.get("content-type", "").lower()
return "application/json" in _response_content_type(response.headers).lower()
def _stream_query_params(stream: bool, starting_after: int | None) -> Mapping[str, object]:
@ -3064,7 +3080,7 @@ class BaseLLMHTTPHandler:
}
# Check if streaming is requested
stream = response_api_optional_request_params.get("stream", False)
stream = _requested_stream(response_api_optional_request_params)
caller_requested_stream: Final = bool(stream or (extra_body or {}).get("stream"))
api_base: Final = responses_api_provider_config.get_complete_url(
@ -4136,7 +4152,7 @@ class BaseLLMHTTPHandler:
):
# Handle pre-signed requests (e.g., from Bedrock S3 uploads)
# Type narrowing: this is a plain dict, not TwoStepFileUploadConfig
presigned_request: Final = cast(dict[str, Any], transformed_request)
presigned_request: Final = cast(TwoStepFileUploadRequest, transformed_request)
upload_response = getattr(sync_httpx_client, presigned_request["method"].lower())(
url=presigned_request["url"],
headers=presigned_request["headers"],
@ -4350,7 +4366,7 @@ class BaseLLMHTTPHandler:
):
# Handle pre-signed requests (e.g., from Bedrock S3 uploads)
# Type narrowing: this is a plain dict, not TwoStepFileUploadConfig
presigned_request: Final = cast(dict[str, Any], transformed_request)
presigned_request: Final = cast(TwoStepFileUploadRequest, transformed_request)
upload_response = await getattr(async_httpx_client, presigned_request["method"].lower())(
url=presigned_request["url"],
headers=presigned_request["headers"],
@ -6129,8 +6145,10 @@ class BaseLLMHTTPHandler:
full_model_name = model
if logging_obj is not None:
agentic_params: Final[Mapping[str, object]] = logging_obj.model_call_details.get("agentic_loop_params", {})
full_model_name = cast(str, agentic_params.get("model", model))
agentic_view: Final[_AgenticLoopParamsView] = {
"agentic_loop_params": logging_obj.model_call_details.get("agentic_loop_params", {})
}
full_model_name = agentic_view["agentic_loop_params"].get("model", model)
optional_params: Final = dict(anthropic_messages_optional_request_params)
optional_params.update(patch.optional_params)
@ -11116,7 +11134,7 @@ class BaseLLMHTTPHandler:
extra_body: dict[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
client: HTTPHandler | AsyncHTTPHandler | None = None,
):
) -> Mapping[str, object]:
if client is None or not isinstance(client, AsyncHTTPHandler):
async_httpx_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders(custom_llm_provider),
@ -11407,7 +11425,7 @@ class BaseLLMHTTPHandler:
extra_body: dict[str, object] | None = None,
timeout: float | httpx.Timeout | None = None,
client: HTTPHandler | AsyncHTTPHandler | None = None,
):
) -> Mapping[str, object]:
if client is None or not isinstance(client, AsyncHTTPHandler):
async_httpx_client = get_async_httpx_client(
llm_provider=litellm.LlmProviders(custom_llm_provider),

22
litellm/llms/dashscope/embed/transformation.py
View file

@ -11,12 +11,13 @@ Endpoint
Docs - https://help.aliyun.com/zh/model-studio/text-embedding-synchronous-api
"""
from typing import Final
from typing import Final, Protocol, runtime_checkable
import httpx
from pydantic import ConfigDict, TypeAdapter, with_config
from typing_extensions import NotRequired, ReadOnly, TypedDict
from litellm.litellm_core_utils.core_helpers import json_contains
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import ensure_api_route
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -29,6 +30,19 @@ from ..common_utils import DashScopeError, resolve_dashscope_openai_api_base
DEFAULT_API_BASE: Final = "https://dashscope.aliyuncs.com/compatible-mode/v1"
_DECODED_JSON: Final = TypeAdapter(object)
@runtime_checkable
class _SupportsGetItem(Protocol):
def __getitem__(self, key: str, /) -> object: ...
def _subscript(value: object, key: str) -> object:
if isinstance(value, _SupportsGetItem):
return value[key]
raise TypeError(f"'{type(value).__name__}' object is not subscriptable")
@with_config(ConfigDict(extra="allow", strict=True))
class _DashScopeUsage(TypedDict, total=False):
@ -149,7 +163,7 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
litellm_params: dict,
) -> EmbeddingResponse:
try:
response_json: Final = raw_response.json()
response_json: Final = _DECODED_JSON.validate_python(raw_response.json())
except Exception as e:
raise DashScopeError(
status_code=raw_response.status_code,
@ -163,8 +177,8 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
original_response=response_json,
)
if "error" in response_json:
error: Final[object] = response_json["error"]
if json_contains(response_json, "error"):
error: Final = _subscript(response_json, "error")
message: Final = error.get("message", str(error)) if isinstance(error, dict) else str(error)
raise DashScopeError(
status_code=raw_response.status_code,

33
litellm/llms/dataforseo/search/transformation.py
View file

@ -4,13 +4,14 @@ Calls DataForSEO SERP API to search the web.
DataForSEO API Reference: https://docs.dataforseo.com/v3/serp/google/organic/live/advanced/?bash
"""
from collections.abc import Iterable, Mapping
from typing import Final, Literal
from collections.abc import Iterable, Mapping, Sized
from typing import Final, Literal, Protocol, runtime_checkable
import httpx
from pydantic import ConfigDict, TypeAdapter
from litellm.constants import DEFAULT_DATAFORSEO_LOCATION_CODE
from litellm.litellm_core_utils.core_helpers import decoded_json_object, json_contains
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.llms.base_llm.search.transformation import (
BaseSearchConfig,
@ -21,6 +22,24 @@ from litellm.secret_managers.main import get_secret_str
_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
_JSON_VALUES: Final = TypeAdapter(Iterable[object], config=ConfigDict(hide_input_in_errors=True))
_DECODED_JSON: Final = TypeAdapter(object)
@runtime_checkable
class _SupportsGetItem(Protocol):
def __getitem__(self, key: str | int, /) -> object: ...
def _subscript(value: object, key: str | int) -> object:
if isinstance(value, _SupportsGetItem):
return value[key]
raise TypeError(f"'{type(value).__name__}' object is not subscriptable")
def _json_len(value: object) -> int:
if isinstance(value, Sized):
return len(value)
raise TypeError(f"object of type '{type(value).__name__}' has no len()")
class DataForSEOSearchConfig(BaseSearchConfig):
@ -193,20 +212,20 @@ class DataForSEOSearchConfig(BaseSearchConfig):
Returns:
SearchResponse with standardized format
"""
response_json: Final = raw_response.json()
response_json: Final = _DECODED_JSON.validate_python(raw_response.json())
# Transform results to SearchResult objects
results: Final = []
# DataForSEO wraps results in tasks array
if "tasks" in response_json and len(response_json["tasks"]) > 0:
task: Final = response_json["tasks"][0]
if json_contains(response_json, "tasks") and _json_len(_subscript(response_json, "tasks")) > 0:
task: Final = decoded_json_object(_subscript(_subscript(response_json, "tasks"), 0))
# Check if task was successful
if task.get("status_code") == 20000 and "result" in task:
# Result is an array, take first element
if len(task["result"]) > 0:
result: Final = _JSON_OBJECT.validate_python(task["result"][0])
if _json_len(task["result"]) > 0:
result: Final = _JSON_OBJECT.validate_python(_subscript(task["result"], 0))
# Items contain the actual search results
for raw_item in _JSON_VALUES.validate_python(result.get("items", [])):

Some files were not shown because too many files have changed in this diff Show more